Skip to content

toolboxv2 API Reference

This section provides an API reference for key components directly available from the toolboxv2 package.

Core Application & Tooling

toolboxv2.AppType

Source code in toolboxv2/utils/system/types.py
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
class AppType:
    prefix: str
    id: str
    globals: dict[str, Any] = {"root": dict, }
    locals: dict[str, Any] = {"user": {'app': "self"}, }

    local_test: bool = False
    start_dir: str
    data_dir: str
    config_dir: str
    info_dir: str
    is_server:bool = False

    logger: logging.Logger
    logging_filename: str

    api_allowed_mods_list: list[str] = []

    version: str
    loop: asyncio.AbstractEventLoop

    keys: dict[str, str] = {
        "MACRO": "macro~~~~:",
        "MACRO_C": "m_color~~:",
        "HELPER": "helper~~~:",
        "debug": "debug~~~~:",
        "id": "name-spa~:",
        "st-load": "mute~load:",
        "comm-his": "comm-his~:",
        "develop-mode": "dev~mode~:",
        "provider::": "provider::",
    }

    defaults: dict[str, (bool or dict or dict[str, dict[str, str]] or str or list[str] or list[list]) | None] = {
        "MACRO": list[str],
        "MACRO_C": dict,
        "HELPER": dict,
        "debug": str,
        "id": str,
        "st-load": False,
        "comm-his": list[list],
        "develop-mode": bool,
    }

    cluster_manager: ClusterManager
    root_blob_storage: BlobStorage
    config_fh: FileHandler
    _debug: bool
    flows: dict[str, Callable]
    dev_modi: bool
    functions: dict[str, Any]
    modules: dict[str, Any]

    interface_type: ToolBoxInterfaces
    REFIX: str

    alive: bool
    called_exit: tuple[bool, float]
    args_sto: AppArgs
    system_flag = None
    session = None
    appdata = None
    exit_tasks = []

    enable_profiling: bool = False
    sto = None

    websocket_handlers: dict[str, dict[str, Callable]] = {}
    _rust_ws_bridge: Any = None

    docs_reader: Callable | None = None
    docs_writer: Callable | None = None
    get_update_suggestions: Callable | None = None
    auto_update_docs: Callable | None = None
    source_code_lookup: Callable | None = None

    initial_docs_parse: Callable | None = None

    def __init__(self, prefix=None, args=None):
        self.args_sto = args
        self.prefix = prefix
        self._footprint_start_time = time.time()
        self._process = psutil.Process(os.getpid())

        # Tracking-Daten für Min/Max/Avg
        self._footprint_metrics = {
            'memory': {'max': 0, 'min': float('inf'), 'samples': []},
            'cpu': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_read': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_write': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_sent': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_recv': {'max': 0, 'min': float('inf'), 'samples': []},
        }

        # Initial Disk/Network Counters
        try:
            io_counters = self._process.io_counters()
            self._initial_disk_read = io_counters.read_bytes
            self._initial_disk_write = io_counters.write_bytes
        except (AttributeError, OSError):
            self._initial_disk_read = 0
            self._initial_disk_write = 0

        try:
            net_io = psutil.net_io_counters()
            self._initial_network_sent = net_io.bytes_sent
            self._initial_network_recv = net_io.bytes_recv
        except (AttributeError, OSError):
            self._initial_network_sent = 0
            self._initial_network_recv = 0

    def _update_metric_tracking(self, metric_name: str, value: float):
        """Aktualisiert Min/Max/Avg für eine Metrik"""
        metrics = self._footprint_metrics[metric_name]
        metrics['max'] = max(metrics['max'], value)
        metrics['min'] = min(metrics['min'], value)
        metrics['samples'].append(value)

        # Begrenze die Anzahl der Samples (letzte 1000)
        if len(metrics['samples']) > 1000:
            metrics['samples'] = metrics['samples'][-1000:]

    def _get_metric_avg(self, metric_name: str) -> float:
        """Berechnet Durchschnitt einer Metrik"""
        samples = self._footprint_metrics[metric_name]['samples']
        return sum(samples) / len(samples) if samples else 0

    def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
        """
        Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

        Args:
            update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

        Returns:
            FootprintMetrics mit allen erfassten Metriken
        """
        current_time = time.time()
        uptime_seconds = current_time - self._footprint_start_time

        # Formatierte Uptime
        uptime_delta = timedelta(seconds=int(uptime_seconds))
        uptime_formatted = str(uptime_delta)

        # Memory Metrics (in MB)
        try:
            mem_info = self._process.memory_info()
            memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
            memory_percent = self._process.memory_percent()

            if update_tracking:
                self._update_metric_tracking('memory', memory_current)

            memory_max = self._footprint_metrics['memory']['max']
            memory_min = self._footprint_metrics['memory']['min']
            if memory_min == float('inf'):
                memory_min = memory_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            memory_current = memory_max = memory_min = memory_percent = 0

        # CPU Metrics
        try:
            cpu_percent_current = self._process.cpu_percent(interval=0.1)
            cpu_times = self._process.cpu_times()
            cpu_time_seconds = cpu_times.user + cpu_times.system

            if update_tracking:
                self._update_metric_tracking('cpu', cpu_percent_current)

            cpu_percent_max = self._footprint_metrics['cpu']['max']
            cpu_percent_min = self._footprint_metrics['cpu']['min']
            cpu_percent_avg = self._get_metric_avg('cpu')

            if cpu_percent_min == float('inf'):
                cpu_percent_min = cpu_percent_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            cpu_percent_current = cpu_percent_max = 0
            cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

        # Disk I/O Metrics (in MB)
        try:
            io_counters = self._process.io_counters()
            disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
            disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

            disk_read_mb = disk_read_bytes / (1024 * 1024)
            disk_write_mb = disk_write_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('disk_read', disk_read_mb)
                self._update_metric_tracking('disk_write', disk_write_mb)

            disk_read_max = self._footprint_metrics['disk_read']['max']
            disk_read_min = self._footprint_metrics['disk_read']['min']
            disk_write_max = self._footprint_metrics['disk_write']['max']
            disk_write_min = self._footprint_metrics['disk_write']['min']

            if disk_read_min == float('inf'):
                disk_read_min = disk_read_mb
            if disk_write_min == float('inf'):
                disk_write_min = disk_write_mb
        except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
            disk_read_mb = disk_write_mb = 0
            disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

        # Network I/O Metrics (in MB)
        try:
            net_io = psutil.net_io_counters()
            network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
            network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

            network_sent_mb = network_sent_bytes / (1024 * 1024)
            network_recv_mb = network_recv_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('network_sent', network_sent_mb)
                self._update_metric_tracking('network_recv', network_recv_mb)

            network_sent_max = self._footprint_metrics['network_sent']['max']
            network_sent_min = self._footprint_metrics['network_sent']['min']
            network_recv_max = self._footprint_metrics['network_recv']['max']
            network_recv_min = self._footprint_metrics['network_recv']['min']

            if network_sent_min == float('inf'):
                network_sent_min = network_sent_mb
            if network_recv_min == float('inf'):
                network_recv_min = network_recv_mb
        except (AttributeError, OSError):
            network_sent_mb = network_recv_mb = 0
            network_sent_max = network_sent_min = 0
            network_recv_max = network_recv_min = 0

        # Process Info
        try:
            process_id = self._process.pid
            threads = self._process.num_threads()
            open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
            connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

            open_files = len(open_files_path)
            connections = len(connections_uri)
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            process_id = os.getpid()
            threads = open_files = connections = 0
            open_files_path = []
            connections_uri = []

        return FootprintMetrics(
            start_time=self._footprint_start_time,
            uptime_seconds=uptime_seconds,
            uptime_formatted=uptime_formatted,
            memory_current=memory_current,
            memory_max=memory_max,
            memory_min=memory_min,
            memory_percent=memory_percent,
            cpu_percent_current=cpu_percent_current,
            cpu_percent_max=cpu_percent_max,
            cpu_percent_min=cpu_percent_min,
            cpu_percent_avg=cpu_percent_avg,
            cpu_time_seconds=cpu_time_seconds,
            disk_read_mb=disk_read_mb,
            disk_write_mb=disk_write_mb,
            disk_read_max=disk_read_max,
            disk_read_min=disk_read_min,
            disk_write_max=disk_write_max,
            disk_write_min=disk_write_min,
            network_sent_mb=network_sent_mb,
            network_recv_mb=network_recv_mb,
            network_sent_max=network_sent_max,
            network_sent_min=network_sent_min,
            network_recv_max=network_recv_max,
            network_recv_min=network_recv_min,
            process_id=process_id,
            threads=threads,
            open_files=open_files,
            connections=connections,
            open_files_path=open_files_path,
            connections_uri=connections_uri,
        )

    def print_footprint(self, detailed: bool = True) -> str:
        """
        Gibt den Footprint formatiert aus.

        Args:
            detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

        Returns:
            Formatierter Footprint-String
        """
        metrics = self.footprint()

        output = [
            "=" * 70,
            f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
            "=" * 70,
            f"\n📊 UPTIME",
            f"  Runtime: {metrics.uptime_formatted}",
            f"  Seconds: {metrics.uptime_seconds:.2f}s",
            f"\n💾 MEMORY USAGE",
            f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
            f"  Maximum:  {metrics.memory_max:.2f} MB",
            f"  Minimum:  {metrics.memory_min:.2f} MB",
        ]

        if detailed:
            helper_ = '\n\t- '.join(metrics.open_files_path)
            helper__ = '\n\t- '.join(metrics.connections_uri)
            output.extend([
                f"\n⚙️  CPU USAGE",
                f"  Current:  {metrics.cpu_percent_current:.2f}%",
                f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
                f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
                f"  Average:  {metrics.cpu_percent_avg:.2f}%",
                f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
                f"\n💿 DISK I/O",
                f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
                f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
                f"\n🌐 NETWORK I/O",
                f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
                f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
                f"\n🔧 PROCESS INFO",
                f"  PID:         {metrics.process_id}",
                f"  Threads:     {metrics.threads}",
                f"\n📂 OPEN FILES",
                f"  Open Files:  {metrics.open_files}",
                f"  Open Files Path: \n\t- {helper_}",
                f"\n🔗 NETWORK CONNECTIONS",
                f"  Connections: {metrics.connections}",
                f"  Connections URI: \n\t- {helper__}",
            ])

        output.append("=" * 70)

        return "\n".join(output)



    def start_server(self):
        from toolboxv2.utils.clis.api import manage_server
        if self.is_server:
            return
        manage_server("start")
        self.is_server = False

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        """proxi attr"""

    @property
    def debug(self):
        """proxi attr"""
        return self._debug

    def debug_rains(self, e):
        """proxi attr"""

    def set_flows(self, r):
        """proxi attr"""

    async def run_flows(self, name, **kwargs):
        """proxi attr"""

    def rrun_flows(self, name, **kwargs):
        """proxi attr"""

    def idle(self):
        import time
        self.print("idle")
        try:
            while self.alive:
                time.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("idle done")

    async def a_idle(self):
        self.print("a idle (running :"+("online)" if hasattr(self, 'daemon_app') else "offline)"))
        try:
            if hasattr(self, 'daemon_app'):
                await self.daemon_app.connect(self)
            else:
                while self.alive:
                    await asyncio.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("a idle done")

    @debug.setter
    def debug(self, value):
        """proxi attr"""

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):
        """proxi attr"""

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        """proxi attr"""

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        """proxi attr"""

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
        """proxi attr"""

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
        """proxi attr"""

    def save_initialized_module(self, tools_class, spec):
        """proxi attr"""

    def mod_online(self, mod_name, installed=False):
        """proxi attr"""

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0):
        """proxi attr"""

    def save_exit(self):
        """proxi attr"""

    def load_mod(self, mod_name: str, mlm='I', **kwargs):
        """proxi attr"""

    async def init_module(self, modular):
        return await self.load_mod(modular)

    async def load_external_mods(self):
        """proxi attr"""

    async def load_all_mods_in_file(self, working_dir="mods"):
        """proxi attr"""

    def get_all_mods(self, working_dir="mods", path_to="./runtime"):
        """proxi attr"""

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    def print_ok(self):
        """proxi attr"""
        self.logger.info("OK")

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        """proxi attr"""

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
        """proxi attr"""

    def remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    def exit(self):
        """proxi attr"""

    def web_context(self) -> str:
        """returns the build index ( toolbox web component )"""

    async def a_exit(self):
        """proxi attr"""

    def save_load(self, modname, spec='app'):
        """proxi attr"""

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """

    def run_a_from_sync(self, function, *args):
        """
        run a async fuction
        """

    def run_bg_task_advanced(self, task, *args, **kwargs):
        """
        proxi attr
        """

    def wait_for_bg_tasks(self, timeout=None):
        """
        proxi attr
        """

    def run_bg_task(self, task):
        """
                run a async fuction
                """
    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        """proxi attr"""

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        """proxi attr"""

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                       args_=None,
                       kwargs_=None,
                       *args, **kwargs):
        """run a function remote via http / https"""

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):
        """proxi attr"""

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):
        """proxi attr"""

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        """proxi attr"""

    @staticmethod
    def print(text, *args, **kwargs):
        """proxi attr"""

    @staticmethod
    def sprint(text, *args, **kwargs):
        """proxi attr"""

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def _register_function(self, module_name, func_name, data):
        """proxi attr"""

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial=False,
                          exit_f=False,
                          test=True,
                          samples=None,
                          state=None,
                          pre_compute=None,
                          post_compute=None,
                          memory_cache=False,
                          file_cache=False,
                          row=False,
                          request_as_kwarg=False,
                          memory_cache_max_size=100,
                          memory_cache_ttl=300,
                          websocket_handler: str | None = None,):
        """proxi attr"""

        # data = {
        #     "type": type_,
        #     "module_name": module_name,
        #     "func_name": func_name,
        #     "level": level,
        #     "restrict_in_virtual_mode": restrict_in_virtual_mode,
        #     "func": func,
        #     "api": api,
        #     "helper": helper,
        #     "version": version,
        #     "initial": initial,
        #     "exit_f": exit_f,
        #     "__module__": func.__module__,
        #     "signature": sig,
        #     "params": params,
        #     "state": (
        #         False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
        #     "do_test": test,
        #     "samples": samples,
        #     "request_as_kwarg": request_as_kwarg,

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str or None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           row=False,
           request_as_kwarg: bool = False,
           state: bool or None = None,
           level: int = 0,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      row=row,
                                      request_as_kwarg=request_as_kwarg,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl)

    def print_functions(self, name=None):


        if not self.functions:
            return

        def helper(_functions):
            for func_name, data in _functions.items():
                if not isinstance(data, dict):
                    continue

                func_type = data.get('type', 'Unknown')
                func_level = 'r' if data['level'] == -1 else data['level']
                api_status = 'Api' if data.get('api', False) else 'Non-Api'

                print(f"  Function: {func_name}{data.get('signature', '()')}; "
                      f"Type: {func_type}, Level: {func_level}, {api_status}")

        if name is not None:
            functions = self.functions.get(name)
            if functions is not None:
                print(f"\nModule: {name}; Type: {functions.get('app_instance_type', 'Unknown')}")
                helper(functions)
                return
        for module, functions in self.functions.items():
            print(f"\nModule: {module}; Type: {functions.get('app_instance_type', 'Unknown')}")
            helper(functions)

    def save_autocompletion_dict(self):
        """proxi attr"""

    def get_autocompletion_dict(self):
        """proxi attr"""

    def get_username(self, get_input=False, default="loot") -> str:
        """proxi attr"""

    def save_registry_as_enums(self, directory: str, filename: str):
        """proxi attr"""

    async def execute_all_functions_(self, m_query='', f_query=''):
        print("Executing all functions")
        from ..extras import generate_test_cases
        all_data = {
            "modular_run": 0,
            "modular_fatal_error": 0,
            "errors": 0,
            "modular_sug": 0,
            "coverage": [],
            "total_coverage": {},
        }
        items = list(self.functions.items()).copy()
        for module_name, functions in items:
            infos = {
                "functions_run": 0,
                "functions_fatal_error": 0,
                "error": 0,
                "functions_sug": 0,
                'calls': {},
                'callse': {},
                "coverage": [0, 0],
            }
            all_data['modular_run'] += 1
            if not module_name.startswith(m_query):
                all_data['modular_sug'] += 1
                continue

            with Spinner(message=f"In {module_name}| "):
                f_items = list(functions.items()).copy()
                for function_name, function_data in f_items:
                    if not isinstance(function_data, dict):
                        continue
                    if not function_name.startswith(f_query):
                        continue
                    test: list = function_data.get('do_test')
                    # print(test, module_name, function_name, function_data)
                    infos["coverage"][0] += 1
                    if test is False:
                        continue

                    with Spinner(message=f"\t\t\t\t\t\tfuction {function_name}..."):
                        params: list = function_data.get('params')
                        sig: signature = function_data.get('signature')
                        state: bool = function_data.get('state')
                        samples: bool = function_data.get('samples')

                        test_kwargs_list = [{}]

                        if params is not None:
                            test_kwargs_list = samples if samples is not None else generate_test_cases(sig=sig)
                            # print(test_kwargs)
                            # print(test_kwargs[0])
                            # test_kwargs = test_kwargs_list[0]
                        # print(module_name, function_name, test_kwargs_list)
                        infos["coverage"][1] += 1
                        for test_kwargs in test_kwargs_list:
                            try:
                                # print(f"test Running {state=} |{module_name}.{function_name}")
                                result = await self.a_run_function((module_name, function_name),
                                                                   tb_run_function_with_state=state,
                                                                   **test_kwargs)
                                if not isinstance(result, Result):
                                    result = Result.ok(result)
                                if result.info.exec_code == 0:
                                    infos['calls'][function_name] = [test_kwargs, str(result)]
                                    infos['functions_sug'] += 1
                                else:
                                    infos['functions_sug'] += 1
                                    infos['error'] += 1
                                    infos['callse'][function_name] = [test_kwargs, str(result)]
                            except Exception as e:
                                infos['functions_fatal_error'] += 1
                                infos['callse'][function_name] = [test_kwargs, str(e)]
                            finally:
                                infos['functions_run'] += 1

                if infos['functions_run'] == infos['functions_sug']:
                    all_data['modular_sug'] += 1
                else:
                    all_data['modular_fatal_error'] += 1
                if infos['error'] > 0:
                    all_data['errors'] += infos['error']

                all_data[module_name] = infos
                if infos['coverage'][0] == 0:
                    c = 0
                else:
                    c = infos['coverage'][1] / infos['coverage'][0]
                all_data["coverage"].append(f"{module_name}:{c:.2f}\n")
        total_coverage = sum([float(t.split(":")[-1]) for t in all_data["coverage"]]) / len(all_data["coverage"])
        print(
            f"\n{all_data['modular_run']=}\n{all_data['modular_sug']=}\n{all_data['modular_fatal_error']=}\n{total_coverage=}")
        d = analyze_data(all_data)
        return Result.ok(data=all_data, data_info=d)

    async def execute_function_test(self, module_name: str, function_name: str,
                                    function_data: dict, test_kwargs: dict,
                                    profiler: cProfile.Profile) -> tuple[bool, str, dict, float]:
        start_time = time.time()
        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            try:
                result = await self.a_run_function(
                    (module_name, function_name),
                    tb_run_function_with_state=function_data.get('state'),
                    **test_kwargs
                )

                if not isinstance(result, Result):
                    result = Result.ok(result)

                success = result.info.exec_code == 0
                execution_time = time.time() - start_time
                return success, str(result), test_kwargs, execution_time
            except Exception as e:
                execution_time = time.time() - start_time
                return False, str(e), test_kwargs, execution_time

    async def process_function(self, module_name: str, function_name: str,
                               function_data: dict, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()
        info = ModuleInfo()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            if not isinstance(function_data, dict):
                return function_name, info

            test = function_data.get('do_test')
            info.coverage[0] += 1

            if test is False:
                return function_name, info

            params = function_data.get('params')
            sig = function_data.get('signature')
            samples = function_data.get('samples')

            test_kwargs_list = [{}] if params is None else (
                samples if samples is not None else generate_test_cases(sig=sig)
            )

            info.coverage[1] += 1

            # Create tasks for all test cases
            tasks = [
                self.execute_function_test(module_name, function_name, function_data, test_kwargs, profiler)
                for test_kwargs in test_kwargs_list
            ]

            # Execute all tests concurrently
            results = await asyncio.gather(*tasks)

            total_execution_time = 0
            for success, result_str, test_kwargs, execution_time in results:
                info.functions_run += 1
                total_execution_time += execution_time

                if success:
                    info.functions_sug += 1
                    info.calls[function_name] = [test_kwargs, result_str]
                else:
                    info.functions_sug += 1
                    info.error += 1
                    info.callse[function_name] = [test_kwargs, result_str]

            info.execution_time = time.time() - start_time
            return function_name, info

    async def process_module(self, module_name: str, functions: dict,
                             f_query: str, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_function(module_name, fname, fdata, profiler)
                    for fname, fdata in functions.items()
                    if fname.startswith(f_query)
                ]

                if not tasks:
                    return module_name, ModuleInfo()

                results = await asyncio.gather(*tasks)

                # Combine results from all functions in the module
                combined_info = ModuleInfo()
                total_execution_time = 0

                for _, info in results:
                    combined_info.functions_run += info.functions_run
                    combined_info.functions_fatal_error += info.functions_fatal_error
                    combined_info.error += info.error
                    combined_info.functions_sug += info.functions_sug
                    combined_info.calls.update(info.calls)
                    combined_info.callse.update(info.callse)
                    combined_info.coverage[0] += info.coverage[0]
                    combined_info.coverage[1] += info.coverage[1]
                    total_execution_time += info.execution_time

                combined_info.execution_time = time.time() - start_time
                return module_name, combined_info

    async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
        """
        Execute all functions with parallel processing and optional profiling.

        Args:
            m_query (str): Module name query filter
            f_query (str): Function name query filter
            enable_profiling (bool): Enable detailed profiling information
        """
        print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

        start_time = time.time()
        stats = ExecutionStats()
        items = list(self.functions.items()).copy()

        # Set up profiling
        self.enable_profiling = enable_profiling
        profiler = cProfile.Profile()

        with profile_section(profiler, enable_profiling):
            # Filter modules based on query
            filtered_modules = [
                (mname, mfuncs) for mname, mfuncs in items
                if mname.startswith(m_query)
            ]

            stats.modular_run = len(filtered_modules)

            # Process all modules concurrently
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_module(mname, mfuncs, f_query, profiler)
                    for mname, mfuncs in filtered_modules
                ]

                results = await asyncio.gather(*tasks)

            # Combine results and calculate statistics
            for module_name, info in results:
                if info.functions_run == info.functions_sug:
                    stats.modular_sug += 1
                else:
                    stats.modular_fatal_error += 1

                stats.errors += info.error

                # Calculate coverage
                coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
                stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

                # Store module info
                stats.__dict__[module_name] = info

            # Calculate total coverage
            total_coverage = (
                sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
                if stats.coverage else 0
            )

            stats.total_execution_time = time.time() - start_time

            # Generate profiling stats if enabled
            if enable_profiling:
                s = io.StringIO()
                ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
                ps.print_stats()
                stats.profiling_data = {
                    'detailed_stats': s.getvalue(),
                    'total_time': stats.total_execution_time,
                    'function_count': stats.modular_run,
                    'successful_functions': stats.modular_sug
                }

            print(
                f"\n{stats.modular_run=}"
                f"\n{stats.modular_sug=}"
                f"\n{stats.modular_fatal_error=}"
                f"\n{total_coverage=}"
                f"\nTotal execution time: {stats.total_execution_time:.2f}s"
            )

            if enable_profiling:
                print("\nProfiling Summary:")
                print(f"{'=' * 50}")
                print("Top 10 time-consuming functions:")
                ps.print_stats(10)

            analyzed_data = analyze_data(stats.__dict__)
            return Result.ok(data=stats.__dict__, data_info=analyzed_data)

debug property writable

proxi attr

a_exit() async

proxi attr

Source code in toolboxv2/utils/system/types.py
1903
1904
async def a_exit(self):
    """proxi attr"""

a_fuction_runner(function, function_data, args, kwargs) async

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py
1968
1969
1970
1971
1972
1973
1974
1975
1976
async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

a_remove_mod(mod_name, spec='app', delete=True) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1894
1895
async def a_remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

a_run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1990
1991
1992
1993
1994
1995
async def a_run_any(self, mod_function_name: Enum or str or tuple,
                    backwords_compability_variabel_string_holder=None,
                    get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                    kwargs_=None,
                    *args, **kwargs):
    """proxi attr"""

a_run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1948
1949
1950
1951
1952
1953
1954
1955
1956
async def a_run_function(self, mod_function_name: Enum or tuple,
                         tb_run_function_with_state=True,
                         tb_run_with_specification='app',
                         args_=None,
                         kwargs_=None,
                         *args,
                         **kwargs) -> Result:

    """proxi attr"""

debug_rains(e)

proxi attr

Source code in toolboxv2/utils/system/types.py
1787
1788
def debug_rains(self, e):
    """proxi attr"""

disconnect(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1775
1776
1777
@staticmethod
async def disconnect(*args, **kwargs):
    """proxi attr"""

execute_all_functions(m_query='', f_query='', enable_profiling=True) async

Execute all functions with parallel processing and optional profiling.

Parameters:

Name Type Description Default
m_query str

Module name query filter

''
f_query str

Function name query filter

''
enable_profiling bool

Enable detailed profiling information

True
Source code in toolboxv2/utils/system/types.py
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
    """
    Execute all functions with parallel processing and optional profiling.

    Args:
        m_query (str): Module name query filter
        f_query (str): Function name query filter
        enable_profiling (bool): Enable detailed profiling information
    """
    print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

    start_time = time.time()
    stats = ExecutionStats()
    items = list(self.functions.items()).copy()

    # Set up profiling
    self.enable_profiling = enable_profiling
    profiler = cProfile.Profile()

    with profile_section(profiler, enable_profiling):
        # Filter modules based on query
        filtered_modules = [
            (mname, mfuncs) for mname, mfuncs in items
            if mname.startswith(m_query)
        ]

        stats.modular_run = len(filtered_modules)

        # Process all modules concurrently
        async with asyncio.Semaphore(mp.cpu_count()):
            tasks = [
                self.process_module(mname, mfuncs, f_query, profiler)
                for mname, mfuncs in filtered_modules
            ]

            results = await asyncio.gather(*tasks)

        # Combine results and calculate statistics
        for module_name, info in results:
            if info.functions_run == info.functions_sug:
                stats.modular_sug += 1
            else:
                stats.modular_fatal_error += 1

            stats.errors += info.error

            # Calculate coverage
            coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
            stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

            # Store module info
            stats.__dict__[module_name] = info

        # Calculate total coverage
        total_coverage = (
            sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
            if stats.coverage else 0
        )

        stats.total_execution_time = time.time() - start_time

        # Generate profiling stats if enabled
        if enable_profiling:
            s = io.StringIO()
            ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
            ps.print_stats()
            stats.profiling_data = {
                'detailed_stats': s.getvalue(),
                'total_time': stats.total_execution_time,
                'function_count': stats.modular_run,
                'successful_functions': stats.modular_sug
            }

        print(
            f"\n{stats.modular_run=}"
            f"\n{stats.modular_sug=}"
            f"\n{stats.modular_fatal_error=}"
            f"\n{total_coverage=}"
            f"\nTotal execution time: {stats.total_execution_time:.2f}s"
        )

        if enable_profiling:
            print("\nProfiling Summary:")
            print(f"{'=' * 50}")
            print("Top 10 time-consuming functions:")
            ps.print_stats(10)

        analyzed_data = analyze_data(stats.__dict__)
        return Result.ok(data=stats.__dict__, data_info=analyzed_data)

exit()

proxi attr

Source code in toolboxv2/utils/system/types.py
1897
1898
def exit(self):
    """proxi attr"""

exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1763
1764
1765
@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""

footprint(update_tracking=True)

Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

Parameters:

Name Type Description Default
update_tracking bool

Wenn True, aktualisiert Min/Max/Avg-Tracking

True

Returns:

Type Description
FootprintMetrics

FootprintMetrics mit allen erfassten Metriken

Source code in toolboxv2/utils/system/types.py
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
    """
    Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

    Args:
        update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

    Returns:
        FootprintMetrics mit allen erfassten Metriken
    """
    current_time = time.time()
    uptime_seconds = current_time - self._footprint_start_time

    # Formatierte Uptime
    uptime_delta = timedelta(seconds=int(uptime_seconds))
    uptime_formatted = str(uptime_delta)

    # Memory Metrics (in MB)
    try:
        mem_info = self._process.memory_info()
        memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
        memory_percent = self._process.memory_percent()

        if update_tracking:
            self._update_metric_tracking('memory', memory_current)

        memory_max = self._footprint_metrics['memory']['max']
        memory_min = self._footprint_metrics['memory']['min']
        if memory_min == float('inf'):
            memory_min = memory_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        memory_current = memory_max = memory_min = memory_percent = 0

    # CPU Metrics
    try:
        cpu_percent_current = self._process.cpu_percent(interval=0.1)
        cpu_times = self._process.cpu_times()
        cpu_time_seconds = cpu_times.user + cpu_times.system

        if update_tracking:
            self._update_metric_tracking('cpu', cpu_percent_current)

        cpu_percent_max = self._footprint_metrics['cpu']['max']
        cpu_percent_min = self._footprint_metrics['cpu']['min']
        cpu_percent_avg = self._get_metric_avg('cpu')

        if cpu_percent_min == float('inf'):
            cpu_percent_min = cpu_percent_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        cpu_percent_current = cpu_percent_max = 0
        cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

    # Disk I/O Metrics (in MB)
    try:
        io_counters = self._process.io_counters()
        disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
        disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

        disk_read_mb = disk_read_bytes / (1024 * 1024)
        disk_write_mb = disk_write_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('disk_read', disk_read_mb)
            self._update_metric_tracking('disk_write', disk_write_mb)

        disk_read_max = self._footprint_metrics['disk_read']['max']
        disk_read_min = self._footprint_metrics['disk_read']['min']
        disk_write_max = self._footprint_metrics['disk_write']['max']
        disk_write_min = self._footprint_metrics['disk_write']['min']

        if disk_read_min == float('inf'):
            disk_read_min = disk_read_mb
        if disk_write_min == float('inf'):
            disk_write_min = disk_write_mb
    except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
        disk_read_mb = disk_write_mb = 0
        disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

    # Network I/O Metrics (in MB)
    try:
        net_io = psutil.net_io_counters()
        network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
        network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

        network_sent_mb = network_sent_bytes / (1024 * 1024)
        network_recv_mb = network_recv_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('network_sent', network_sent_mb)
            self._update_metric_tracking('network_recv', network_recv_mb)

        network_sent_max = self._footprint_metrics['network_sent']['max']
        network_sent_min = self._footprint_metrics['network_sent']['min']
        network_recv_max = self._footprint_metrics['network_recv']['max']
        network_recv_min = self._footprint_metrics['network_recv']['min']

        if network_sent_min == float('inf'):
            network_sent_min = network_sent_mb
        if network_recv_min == float('inf'):
            network_recv_min = network_recv_mb
    except (AttributeError, OSError):
        network_sent_mb = network_recv_mb = 0
        network_sent_max = network_sent_min = 0
        network_recv_max = network_recv_min = 0

    # Process Info
    try:
        process_id = self._process.pid
        threads = self._process.num_threads()
        open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
        connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

        open_files = len(open_files_path)
        connections = len(connections_uri)
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        process_id = os.getpid()
        threads = open_files = connections = 0
        open_files_path = []
        connections_uri = []

    return FootprintMetrics(
        start_time=self._footprint_start_time,
        uptime_seconds=uptime_seconds,
        uptime_formatted=uptime_formatted,
        memory_current=memory_current,
        memory_max=memory_max,
        memory_min=memory_min,
        memory_percent=memory_percent,
        cpu_percent_current=cpu_percent_current,
        cpu_percent_max=cpu_percent_max,
        cpu_percent_min=cpu_percent_min,
        cpu_percent_avg=cpu_percent_avg,
        cpu_time_seconds=cpu_time_seconds,
        disk_read_mb=disk_read_mb,
        disk_write_mb=disk_write_mb,
        disk_read_max=disk_read_max,
        disk_read_min=disk_read_min,
        disk_write_max=disk_write_max,
        disk_write_min=disk_write_min,
        network_sent_mb=network_sent_mb,
        network_recv_mb=network_recv_mb,
        network_sent_max=network_sent_max,
        network_sent_min=network_sent_min,
        network_recv_max=network_recv_max,
        network_recv_min=network_recv_min,
        process_id=process_id,
        threads=threads,
        open_files=open_files,
        connections=connections,
        open_files_path=open_files_path,
        connections_uri=connections_uri,
    )

fuction_runner(function, function_data, args, kwargs, t0=0.0)

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py
1958
1959
1960
1961
1962
1963
1964
1965
1966
def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """

get_all_mods(working_dir='mods', path_to='./runtime')

proxi attr

Source code in toolboxv2/utils/system/types.py
1868
1869
def get_all_mods(self, working_dir="mods", path_to="./runtime"):
    """proxi attr"""

get_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py
2174
2175
def get_autocompletion_dict(self):
    """proxi attr"""

get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/system/types.py
1909
1910
1911
1912
1913
1914
1915
1916
1917
def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """

get_mod(name, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py
1997
1998
def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
    """proxi attr"""

get_username(get_input=False, default='loot')

proxi attr

Source code in toolboxv2/utils/system/types.py
2177
2178
def get_username(self, get_input=False, default="loot") -> str:
    """proxi attr"""

hide_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1767
1768
1769
@staticmethod
async def hide_console(*args, **kwargs):
    """proxi attr"""

inplace_load_instance(mod_name, loc='toolboxv2.mods.', spec='app', save=True)

proxi attr

Source code in toolboxv2/utils/system/types.py
1834
1835
def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
    """proxi attr"""

load_all_mods_in_file(working_dir='mods') async

proxi attr

Source code in toolboxv2/utils/system/types.py
1865
1866
async def load_all_mods_in_file(self, working_dir="mods"):
    """proxi attr"""

load_external_mods() async

proxi attr

Source code in toolboxv2/utils/system/types.py
1862
1863
async def load_external_mods(self):
    """proxi attr"""

load_mod(mod_name, mlm='I', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1856
1857
def load_mod(self, mod_name: str, mlm='I', **kwargs):
    """proxi attr"""

mod_online(mod_name, installed=False)

proxi attr

Source code in toolboxv2/utils/system/types.py
1843
1844
def mod_online(self, mod_name, installed=False):
    """proxi attr"""

print(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
2000
2001
2002
@staticmethod
def print(text, *args, **kwargs):
    """proxi attr"""

print_footprint(detailed=True)

Gibt den Footprint formatiert aus.

Parameters:

Name Type Description Default
detailed bool

Wenn True, zeigt alle Details, sonst nur Zusammenfassung

True

Returns:

Type Description
str

Formatierter Footprint-String

Source code in toolboxv2/utils/system/types.py
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
def print_footprint(self, detailed: bool = True) -> str:
    """
    Gibt den Footprint formatiert aus.

    Args:
        detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

    Returns:
        Formatierter Footprint-String
    """
    metrics = self.footprint()

    output = [
        "=" * 70,
        f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
        "=" * 70,
        f"\n📊 UPTIME",
        f"  Runtime: {metrics.uptime_formatted}",
        f"  Seconds: {metrics.uptime_seconds:.2f}s",
        f"\n💾 MEMORY USAGE",
        f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
        f"  Maximum:  {metrics.memory_max:.2f} MB",
        f"  Minimum:  {metrics.memory_min:.2f} MB",
    ]

    if detailed:
        helper_ = '\n\t- '.join(metrics.open_files_path)
        helper__ = '\n\t- '.join(metrics.connections_uri)
        output.extend([
            f"\n⚙️  CPU USAGE",
            f"  Current:  {metrics.cpu_percent_current:.2f}%",
            f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
            f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
            f"  Average:  {metrics.cpu_percent_avg:.2f}%",
            f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
            f"\n💿 DISK I/O",
            f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
            f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
            f"\n🌐 NETWORK I/O",
            f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
            f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
            f"\n🔧 PROCESS INFO",
            f"  PID:         {metrics.process_id}",
            f"  Threads:     {metrics.threads}",
            f"\n📂 OPEN FILES",
            f"  Open Files:  {metrics.open_files}",
            f"  Open Files Path: \n\t- {helper_}",
            f"\n🔗 NETWORK CONNECTIONS",
            f"  Connections: {metrics.connections}",
            f"  Connections URI: \n\t- {helper__}",
        ])

    output.append("=" * 70)

    return "\n".join(output)

print_ok()

proxi attr

Source code in toolboxv2/utils/system/types.py
1881
1882
1883
def print_ok(self):
    """proxi attr"""
    self.logger.info("OK")

reload_mod(mod_name, spec='app', is_file=True, loc='toolboxv2.mods.')

proxi attr

Source code in toolboxv2/utils/system/types.py
1885
1886
def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
    """proxi attr"""

remove_mod(mod_name, spec='app', delete=True)

proxi attr

Source code in toolboxv2/utils/system/types.py
1891
1892
def remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""

rrun_flows(name, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1796
1797
def rrun_flows(self, name, **kwargs):
    """proxi attr"""

run_a_from_sync(function, *args)

run a async fuction

Source code in toolboxv2/utils/system/types.py
1919
1920
1921
1922
def run_a_from_sync(self, function, *args):
    """
    run a async fuction
    """

run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1984
1985
1986
1987
1988
def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
            get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
            kwargs_=None,
            *args, **kwargs):
    """proxi attr"""

run_bg_task(task)

run a async fuction

Source code in toolboxv2/utils/system/types.py
1934
1935
1936
1937
def run_bg_task(self, task):
    """
            run a async fuction
            """

run_bg_task_advanced(task, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1924
1925
1926
1927
def run_bg_task_advanced(self, task, *args, **kwargs):
    """
    proxi attr
    """

run_flows(name, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1793
1794
async def run_flows(self, name, **kwargs):
    """proxi attr"""

run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1938
1939
1940
1941
1942
1943
1944
1945
1946
def run_function(self, mod_function_name: Enum or tuple,
                 tb_run_function_with_state=True,
                 tb_run_with_specification='app',
                 args_=None,
                 kwargs_=None,
                 *args,
                 **kwargs) -> Result:

    """proxi attr"""

run_http(mod_function_name, function_name=None, method='GET', args_=None, kwargs_=None, *args, **kwargs) async

run a function remote via http / https

Source code in toolboxv2/utils/system/types.py
1978
1979
1980
1981
1982
async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                   args_=None,
                   kwargs_=None,
                   *args, **kwargs):
    """run a function remote via http / https"""

save_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py
2171
2172
def save_autocompletion_dict(self):
    """proxi attr"""

save_exit()

proxi attr

Source code in toolboxv2/utils/system/types.py
1853
1854
def save_exit(self):
    """proxi attr"""

save_initialized_module(tools_class, spec)

proxi attr

Source code in toolboxv2/utils/system/types.py
1840
1841
def save_initialized_module(self, tools_class, spec):
    """proxi attr"""

save_instance(instance, modular_id, spec='app', instance_type='file/application', tools_class=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1837
1838
def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
    """proxi attr"""

save_load(modname, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py
1906
1907
def save_load(self, modname, spec='app'):
    """proxi attr"""

save_registry_as_enums(directory, filename)

proxi attr

Source code in toolboxv2/utils/system/types.py
2180
2181
def save_registry_as_enums(self, directory: str, filename: str):
    """proxi attr"""

set_flows(r)

proxi attr

Source code in toolboxv2/utils/system/types.py
1790
1791
def set_flows(self, r):
    """proxi attr"""

set_logger(debug=False)

proxi attr

Source code in toolboxv2/utils/system/types.py
1779
1780
def set_logger(self, debug=False):
    """proxi attr"""

show_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1771
1772
1773
@staticmethod
async def show_console(*args, **kwargs):
    """proxi attr"""

sprint(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
2004
2005
2006
@staticmethod
def sprint(text, *args, **kwargs):
    """proxi attr"""

tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, row=False, request_as_kwarg=False, state=None, level=0, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name Type Description Default
name str

The name to register the function under. Defaults to the function's own name.

None
mod_name str

The name of the module the function belongs to.

''
helper str

A helper string providing additional information about the function.

''
version str or None

The version of the function or module.

None
test bool

Flag to indicate if the function is for testing purposes.

True
restrict_in_virtual_mode bool

Flag to restrict the function in virtual mode.

False
api bool

Flag to indicate if the function is part of an API.

False
initial bool

Flag to indicate if the function should be executed at initialization.

False
exit_f bool

Flag to indicate if the function should be executed at exit.

False
test_only bool

Flag to indicate if the function should only be used for testing.

False
memory_cache bool

Flag to enable memory caching for the function.

False
request_as_kwarg bool

Flag to get request if the fuction is calld from api.

False
file_cache bool

Flag to enable file caching for the function.

False
row bool

rather to auto wrap the result in Result type default False means no row data aka result type

False
state bool or None

Flag to indicate if the function maintains state.

None
level int

The level of the function, used for prioritization or categorization.

0
memory_cache_max_size int

Maximum size of the memory cache.

100
memory_cache_ttl int

Time-to-live for the memory cache entries.

300
samples list or dict or None

Samples or examples of function usage.

None
interface str

The interface type for the function.

None
pre_compute callable

A function to be called before the main function.

None
post_compute callable

A function to be called after the main function.

None
api_methods list[str]

default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

None

Returns:

Name Type Description
function

The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/system/types.py
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str or None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       row=False,
       request_as_kwarg: bool = False,
       state: bool or None = None,
       level: int = 0,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  row=row,
                                  request_as_kwarg=request_as_kwarg,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl)

wait_for_bg_tasks(timeout=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1929
1930
1931
1932
def wait_for_bg_tasks(self, timeout=None):
    """
    proxi attr
    """

watch_mod(mod_name, spec='app', loc='toolboxv2.mods.', use_thread=True, path_name=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1888
1889
def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
    """proxi attr"""

web_context()

returns the build index ( toolbox web component )

Source code in toolboxv2/utils/system/types.py
1900
1901
def web_context(self) -> str:
    """returns the build index ( toolbox web component )"""

toolboxv2.MainTool

Source code in toolboxv2/utils/system/main_tool.py
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")

__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py
174
175
176
177
178
179
180
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py
167
168
169
def get_version(self) -> str:
    """"Returns the version"""
    return self.version

webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py
164
165
def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

toolboxv2.get_app(from_=None, name=None, args=AppArgs().default(), app_con=None, sync=False)

Source code in toolboxv2/utils/system/getting_and_closing_app.py
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def get_app(from_=None, name=None, args=AppArgs().default(), app_con=None, sync=False) -> AppType:
    global registered_apps
    # name = None
    # inspect caller
    # from inspect import getouterframes, currentframe
    # print(f"get app requested from: {getouterframes(currentframe(), 2)[1].filename}::{getouterframes(currentframe(), 2)[1].lineno}")

    # print(f"get app requested from: {from_} withe name: {name}")
    logger = get_logger()
    logger.info(Style.GREYBG(f"get app requested from: {from_}"))
    if registered_apps[0] is not None:
        return registered_apps[0]

    if app_con is None:
        from ... import App
        app_con = App
    app = app_con(name, args=args) if name else app_con()
    logger.info(Style.Bold(f"App instance, returned ID: {app.id}"))

    registered_apps[0] = app
    return app

System Utilities & Configuration

toolboxv2.FileHandler

Bases: Code

Source code in toolboxv2/utils/system/file_handler.py
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
class FileHandler(Code):

    def __init__(self, filename, name='mainTool', keys=None, defaults=None):
        if defaults is None:
            defaults = {}
        if keys is None:
            keys = {}
        assert filename.endswith(".config") or filename.endswith(".data"), \
            f"filename must end with .config or .data {filename=}"
        self.file_handler_save = {}
        self.file_handler_load = {}
        self.file_handler_key_mapper = {}
        self.file_handler_filename = filename
        self.file_handler_storage = None
        self.file_handler_max_loaded_index_ = 0
        self.file_handler_file_prefix = (f".{filename.split('.')[1]}/"
                                         f"{name.replace('.', '-')}/")
        # self.load_file_handler()
        self.set_defaults_keys_file_handler(keys, defaults)

    def _open_file_handler(self, mode: str, rdu):
        logger = get_logger()
        logger.info(Style.Bold(Style.YELLOW(f"Opening file in mode : {mode}")))
        if self.file_handler_storage:
            self.file_handler_storage.close()
            self.file_handler_storage = None
        try:
            self.file_handler_storage = open(self.file_handler_file_prefix + self.file_handler_filename, mode)
            self.file_handler_max_loaded_index_ += 1
        except FileNotFoundError:
            if self.file_handler_max_loaded_index_ == 2:
                os.makedirs(self.file_handler_file_prefix, exist_ok=True)
            if self.file_handler_max_loaded_index_ == 3:
                os.makedirs(".config/mainTool", exist_ok=True)
            if self.file_handler_max_loaded_index_ >= 5:
                print(Style.RED(f"pleas create this file to prosed : {self.file_handler_file_prefix}"
                                f"{self.file_handler_filename}"))
                logger.critical(f"{self.file_handler_file_prefix} {self.file_handler_filename} FileNotFoundError cannot"
                                f" be Created")
                exit(0)
            self.file_handler_max_loaded_index_ += 1
            logger.info(Style.YELLOW(f"Try Creating File: {self.file_handler_file_prefix}{self.file_handler_filename}"))

            if not os.path.exists(f"{self.file_handler_file_prefix}"):
                os.makedirs(f"{self.file_handler_file_prefix}")

            with open(self.file_handler_file_prefix + self.file_handler_filename, 'a'):
                logger.info(Style.GREEN("File created successfully"))
                self.file_handler_max_loaded_index_ = -1
            rdu()
        except OSError and PermissionError as e:
            raise e

    def open_s_file_handler(self):
        self._open_file_handler('w+', self.open_s_file_handler)
        return self

    def open_l_file_handler(self):
        self._open_file_handler('r+', self.open_l_file_handler)
        return self

    def save_file_handler(self):
        get_logger().info(
            Style.BLUE(
                f"init Saving (S) {self.file_handler_filename} "
            )
        )
        if self.file_handler_storage:
            get_logger().warning(
                f"WARNING file is already open (S): {self.file_handler_filename} {self.file_handler_storage}")

        self.open_s_file_handler()

        get_logger().info(
            Style.BLUE(
                f"Elements to save : ({len(self.file_handler_save.keys())})"
            )
        )

        self.file_handler_storage.write(json.dumps(self.file_handler_save))

        self.file_handler_storage.close()
        self.file_handler_storage = None

        get_logger().info(
            Style.BLUE(
                f"closing file : {self.file_handler_filename} "
            )
        )

        return self

    def add_to_save_file_handler(self, key: str, value: str):
        if len(key) != 10:
            get_logger(). \
                warning(
                Style.YELLOW(
                    'WARNING: key length is not 10 characters'
                )
            )
            return False
        if key not in self.file_handler_load:
            if key in self.file_handler_key_mapper:
                key = self.file_handler_key_mapper[key]

        self.file_handler_load[key] = value
        self.file_handler_save[key] = self.encode_code(value)
        return True

    def remove_key_file_handler(self, key: str):
        if key == 'Pka7237327':
            print("Cant remove Root Key")
            return
        if key in self.file_handler_load:
            del self.file_handler_load[key]
        if key in self.file_handler_save:
            del self.file_handler_save[key]

    def load_file_handler(self):
        get_logger().info(
            Style.BLUE(
                f"loading {self.file_handler_filename} "
            )
        )
        if self.file_handler_storage:
            get_logger().warning(
                Style.YELLOW(
                    f"WARNING file is already open (L) {self.file_handler_filename}"
                )
            )
        self.open_l_file_handler()

        try:

            self.file_handler_save = json.load(self.file_handler_storage)
            for key, line in self.file_handler_save.items():
                self.file_handler_load[key] = self.decode_code(line)

        except json.decoder.JSONDecodeError and Exception:

            for line in self.file_handler_storage:
                line = line[:-1]
                heda = line[:10]
                self.file_handler_save[heda] = line[10:]
                enc = self.decode_code(line[10:])
                self.file_handler_load[heda] = enc

            self.file_handler_save = {}

        self.file_handler_storage.close()
        self.file_handler_storage = None

        return self

    def get_file_handler(self, obj: str, default=None) -> str or None:
        logger = get_logger()
        if obj not in self.file_handler_load:
            if obj in self.file_handler_key_mapper:
                obj = self.file_handler_key_mapper[obj]
        logger.info(Style.ITALIC(Style.GREY(f"Collecting data from storage key : {obj}")))
        self.file_handler_max_loaded_index_ = -1
        for objects in self.file_handler_load.items():
            self.file_handler_max_loaded_index_ += 1
            if obj == objects[0]:

                try:
                    if len(objects[1]) > 0:
                        return ast.literal_eval(objects[1]) if isinstance(objects[1], str) else objects[1]
                    logger.warning(
                        Style.YELLOW(
                            f"No data  {obj}  ; {self.file_handler_filename}"
                        )
                    )
                except ValueError:
                    logger.error(f"ValueError Loading {obj} ; {self.file_handler_filename}")
                except SyntaxError:
                    if isinstance(objects[1], str):
                        return objects[1]
                    logger.warning(
                        Style.YELLOW(
                            f"Possible SyntaxError Loading {obj} ; {self.file_handler_filename}"
                            f" {len(objects[1])} {type(objects[1])}"
                        )
                    )
                    return objects[1]
                except NameError:
                    return str(objects[1])

        if obj in list(self.file_handler_save.keys()):
            r = self.decode_code(self.file_handler_save[obj])
            logger.info(f"returning Default for {obj}")
            return r

        if default is None:
            default = self.file_handler_load.get(obj)

        logger.info("no data found")
        return default

    def set_defaults_keys_file_handler(self, keys: dict, defaults: dict):
        list_keys = iter(list(keys.keys()))
        df_keys = defaults.keys()
        for key in list_keys:
            self.file_handler_key_mapper[key] = keys[key]
            self.file_handler_key_mapper[keys[key]] = key
            if key in df_keys:
                self.file_handler_load[keys[key]] = str(defaults[key])
                self.file_handler_save[keys[key]] = self.encode_code(defaults[key])
            else:
                self.file_handler_load[keys[key]] = "None"

    def delete_file(self):
        os.remove(self.file_handler_file_prefix + self.file_handler_filename)
        get_logger().warning(Style.GREEN(f"File deleted {self.file_handler_file_prefix + self.file_handler_filename}"))

toolboxv2.utils

App

Source code in toolboxv2/utils/toolbox.py
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
class App(AppType, metaclass=Singleton):

    def __init__(self, prefix: str = "", args=AppArgs().default()):
        if "test" not in prefix:
            prefix = "main"
        super().__init__(prefix, args)
        self._web_context = None
        t0 = time.perf_counter()
        abspath = os.path.abspath(__file__)
        self.system_flag = system()  # Linux: Linux Mac: Darwin Windows: Windows

        self.appdata = os.getenv('APPDATA') if os.name == 'nt' else os.getenv('XDG_CONFIG_HOME') or os.path.expanduser(
                '~/.config') if os.name == 'posix' else None

        if self.system_flag == "Darwin" or self.system_flag == "Linux":
            dir_name = os.path.dirname(abspath).replace("/utils", "")
        else:
            dir_name = os.path.dirname(abspath).replace("\\utils", "")

        self.start_dir = str(dir_name)

        self.bg_tasks = []

        lapp = dir_name + '\\.data\\'

        if not prefix:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt") as prefix_file:
                cont = prefix_file.read()
                if cont:
                    prefix = cont.rstrip()
        else:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt", "w") as prefix_file:
                prefix_file.write(prefix)

        self.prefix = prefix

        node_ = node()

        if 'localhost' in node_ and (host := os.getenv('HOSTNAME', 'localhost')) != 'localhost':
            node_ = node_.replace('localhost', host)
        self.id = prefix + '-' + node_
        self.globals = {
            "root": {**globals()},
        }
        self.locals = {
            "user": {'app': self, **locals()},
        }

        identification = self.id
        collective_identification = self.id
        if "test" in prefix:
            if self.system_flag == "Darwin" or self.system_flag == "Linux":
                start_dir = self.start_dir.replace("ToolBoxV2/toolboxv2", "toolboxv2")
            else:
                start_dir = self.start_dir.replace("ToolBoxV2\\toolboxv2", "toolboxv2")
            self.data_dir = start_dir + '\\.data\\' + "test"
            self.config_dir = start_dir + '\\.config\\' + "test"
            self.info_dir = start_dir + '\\.info\\' + "test"
        elif identification.startswith('collective-'):
            collective_identification = identification.split('-')[1]
            self.data_dir = self.start_dir + '\\.data\\' + collective_identification
            self.config_dir = self.start_dir + '\\.config\\' + collective_identification
            self.info_dir = self.start_dir + '\\.info\\' + collective_identification
            self.id = collective_identification
        else:
            self.data_dir = self.start_dir + '\\.data\\' + identification
            self.config_dir = self.start_dir + '\\.config\\' + identification
            self.info_dir = self.start_dir + '\\.info\\' + identification

        if self.appdata is None:
            self.appdata = self.data_dir
        else:
            self.appdata += "/ToolBoxV2"

        if not os.path.exists(self.appdata):
            os.makedirs(self.appdata, exist_ok=True)
        if not os.path.exists(self.data_dir):
            os.makedirs(self.data_dir, exist_ok=True)
        if not os.path.exists(self.config_dir):
            os.makedirs(self.config_dir, exist_ok=True)
        if not os.path.exists(self.info_dir):
            os.makedirs(self.info_dir, exist_ok=True)

        self.print(f"Starting ToolBox as {prefix} from :", Style.Bold(Style.CYAN(f"{os.getcwd()}")))

        logger_info_str, self.logger, self.logging_filename = self.set_logger(args.debug)

        self.print("Logger " + logger_info_str)
        self.print("================================")
        self.logger.info("Logger initialized")
        get_logger().info(Style.GREEN("Starting Application instance"))
        if args.init and args.init is not None and self.start_dir not in sys.path:
            sys.path.append(self.start_dir)

        __version__ = get_version_from_pyproject()
        self.version = __version__

        self.keys = {
            "MACRO": "macro~~~~:",
            "MACRO_C": "m_color~~:",
            "HELPER": "helper~~~:",
            "debug": "debug~~~~:",
            "id": "name-spa~:",
            "st-load": "mute~load:",
            "comm-his": "comm-his~:",
            "develop-mode": "dev~mode~:",
            "provider::": "provider::",
        }

        defaults = {
            "MACRO": ['Exit'],
            "MACRO_C": {},
            "HELPER": {},
            "debug": args.debug,
            "id": self.id,
            "st-load": False,
            "comm-his": [[]],
            "develop-mode": False,
        }
        self.config_fh = FileHandler(collective_identification + ".config", keys=self.keys, defaults=defaults)
        self.config_fh.load_file_handler()
        self._debug = args.debug
        self.flows = {}
        self.dev_modi = self.config_fh.get_file_handler(self.keys["develop-mode"])
        if self.config_fh.get_file_handler("provider::") is None:
            self.config_fh.add_to_save_file_handler("provider::", "http://localhost:" + str(
                self.args_sto.port) if os.environ.get("HOSTNAME","localhost") == "localhost" else "https://simplecore.app")
        self.functions = {}
        self.modules = {}

        self.interface_type = ToolBoxInterfaces.native
        self.PREFIX = Style.CYAN(f"~{node()}@>")
        self.alive = True
        self.called_exit = False, time.time()

        self.print(f"Infos:\n  {'Name':<8} -> {node()}\n  {'ID':<8} -> {self.id}\n  {'Version':<8} -> {self.version}\n")

        self.logger.info(
            Style.GREEN(
                f"Finish init up in {time.perf_counter() - t0:.2f}s"
            )
        )

        self.args_sto = args
        self.loop = None

        from .system.session import Session
        self.session: Session = Session(self.get_username())
        if len(sys.argv) > 2 and sys.argv[1] == "db":
            return
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager, get_executable_path
        self.cluster_manager = ClusterManager()
        online_list, server_list = self.cluster_manager.status_all(silent=True)
        if not server_list:
            self.cluster_manager.start_all(get_executable_path(), self.version)
            _, server_list = self.cluster_manager.status_all()
        from .extras.blobs import BlobStorage
        self.root_blob_storage = BlobStorage(servers=server_list, storage_directory=self.data_dir+ '\\blob_cache\\')
        self.mkdocs = add_to_app(self)
        # self._start_event_loop()

    def _start_event_loop(self):
        """Starts the asyncio event loop in a separate thread."""
        if self.loop is None:
            self.loop = asyncio.new_event_loop()
            self.loop_thread = threading.Thread(target=self.loop.run_forever, daemon=True)
            self.loop_thread.start()

    def get_username(self, get_input=False, default="loot") -> str:
        user_name = self.config_fh.get_file_handler("ac_user:::")
        if get_input and user_name is None:
            user_name = input("Input your username: ")
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        if user_name is None:
            user_name = default
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        return user_name

    def set_username(self, username):
        return self.config_fh.add_to_save_file_handler("ac_user:::", username)

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        if "test" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.NOTSET, name="toolbox-test", interminal=True,
                                                     file_level=logging.NOTSET, app_name=self.id)
            logger_info_str = "in Test Mode"
        elif "live" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-live", interminal=False,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in Live Mode"
            # setup_logging(logging.WARNING, name="toolbox-live", is_online=True
            #              , online_level=logging.WARNING).info("Logger initialized")
        elif "debug" in self.prefix or self.prefix.endswith("D"):
            self.prefix = self.prefix.replace("-debug", '').replace("debug", '')
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-debug", interminal=True,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in debug Mode"
            self.debug = True
        elif debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name=f"toolbox-{self.prefix}-debug",
                                                     interminal=True,
                                                     file_level=logging.DEBUG, app_name=self.id)
            logger_info_str = "in args debug Mode"
        else:
            logger, logging_filename = setup_logging(logging.ERROR, name=f"toolbox-{self.prefix}", app_name=self.id)
            logger_info_str = "in Default"

        return logger_info_str, logger, logging_filename

    @property
    def debug(self):
        return self._debug

    @debug.setter
    def debug(self, value):
        if not isinstance(value, bool):
            self.logger.debug(f"Value must be an boolean. is : {value} type of {type(value)}")
            raise ValueError("Value must be an boolean.")

        # self.logger.info(f"Setting debug {value}")
        self._debug = value

    def debug_rains(self, e):
        if self.debug:
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)
            raise e
        else:
            self.logger.error(f"Error: {e}")
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)

    def set_flows(self, r):
        self.flows = r

    async def run_flows(self, name, **kwargs):
        from ..flows import flows_dict as flows_dict_func
        if name not in self.flows:
            self.flows = {**self.flows, **flows_dict_func(s=name, remote=True)}
        if name in self.flows:
            if asyncio.iscoroutinefunction(self.flows[name]):
                return await self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
            else:
                return self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
        else:
            print("Flow not found, active flows:", len(self.flows.keys()))

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):

        mode = 'xb'
        self.logger.info(f" coppy mod {mod_name} to {new_mod_dir} size : {sys.getsizeof(content) / 8388608:.3f} mb")

        if not os.path.exists(new_mod_dir):
            os.makedirs(new_mod_dir)
            with open(f"{new_mod_dir}/__init__.py", "w") as nmd:
                nmd.write(f"__version__ = '{self.version}'")

        if os.path.exists(f"{new_mod_dir}/{mod_name}.{file_type}"):
            mode = False

            with open(f"{new_mod_dir}/{mod_name}.{file_type}", 'rb') as d:
                runtime_mod = d.read()  # Testing version but not efficient

            if len(content) != len(runtime_mod):
                mode = 'wb'

        if mode:
            with open(f"{new_mod_dir}/{mod_name}.{file_type}", mode) as f:
                f.write(content)

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        working_dir = self.id.replace(".", "_")
        lib_mod_dir = f"toolboxv2.runtime.{working_dir}.mod_lib."

        self.logger.info(f"pre_lib_mod {mod_name} from {lib_mod_dir}")

        postfix = "_dev" if self.dev_modi else ""
        mod_file_dir = f"./mods{postfix}/{mod_name}.{file_type}"
        new_mod_dir = f"{path_to}/{working_dir}/mod_lib"
        with open(mod_file_dir, "rb") as c:
            content = c.read()
        self._coppy_mod(content, new_mod_dir, mod_name, file_type=file_type)
        return lib_mod_dir

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        loc = self._pre_lib_mod(mod_name, file_type)
        return self.inplace_load_instance(mod_name, loc=loc, **kwargs)

    def helper_install_pip_module(self, module_name):
        if 'main' in self.id:
            return
        self.print(f"Installing {module_name} GREEDY")
        os.system(f"{sys.executable} -m pip install {module_name}")

    def python_module_import_classifier(self, mod_name, error_message):

        if error_message.startswith("No module named 'toolboxv2.utils"):
            return Result.default_internal_error(f"404 {error_message.split('utils')[1]} not found")
        if error_message.startswith("No module named 'toolboxv2.mods"):
            if mod_name.startswith('.'):
                return
            return self.run_a_from_sync(self.a_run_any, ("CloudM", "install"), module_name=mod_name)
        if error_message.startswith("No module named '"):
            pip_requ = error_message.split("'")[1].replace("'", "").strip()
            # if 'y' in input(f"\t\t\tAuto install {pip_requ} Y/n").lower:
            return self.helper_install_pip_module(pip_requ)
            # return Result.default_internal_error(f"404 {pip_requ} not found")

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True, mfo=None):
        if self.dev_modi and loc == "toolboxv2.mods.":
            loc = "toolboxv2.mods_dev."
        if spec=='app' and self.mod_online(mod_name):
            self.logger.info(f"Reloading mod from : {loc + mod_name}")
            self.remove_mod(mod_name, spec=spec, delete=False)

        if (os.path.exists(self.start_dir + '/mods/' + mod_name) or os.path.exists(
            self.start_dir + '/mods/' + mod_name + '.py')) and (
            os.path.isdir(self.start_dir + '/mods/' + mod_name) or os.path.isfile(
            self.start_dir + '/mods/' + mod_name + '.py')):
            try:
                if mfo is None:
                    modular_file_object = import_module(loc + mod_name)
                else:
                    modular_file_object = mfo
                self.modules[mod_name] = modular_file_object
            except ModuleNotFoundError as e:
                self.logger.error(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                self.print(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                if self.debug or self.args_sto.sysPrint:
                    self.python_module_import_classifier(mod_name, str(e))
                self.debug_rains(e)
                return None
        else:
            self.sprint(f"module {loc + mod_name} is not valid")
            return None
        if hasattr(modular_file_object, "Tools"):
            tools_class = modular_file_object.Tools
        else:
            if hasattr(modular_file_object, "name"):
                tools_class = modular_file_object
                modular_file_object = import_module(loc + mod_name)
            else:
                tools_class = None

        modular_id = None
        instance = modular_file_object
        app_instance_type = "file/application"

        if tools_class is None:
            modular_id = modular_file_object.Name if hasattr(modular_file_object, "Name") else mod_name

        if tools_class is None and modular_id is None:
            modular_id = str(modular_file_object.__name__)
            self.logger.warning(f"Unknown instance loaded {mod_name}")
            return modular_file_object

        if tools_class is not None:
            tools_class = self.save_initialized_module(tools_class, spec)
            modular_id = tools_class.name
            app_instance_type = "functions/class"
        else:
            instance.spec = spec
        # if private:
        #     self.functions[modular_id][f"{spec}_private"] = private

        if not save:
            return instance if tools_class is None else tools_class

        return self.save_instance(instance, modular_id, spec, app_instance_type, tools_class=tools_class)

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):

        if modular_id in self.functions and tools_class is None:
            if self.functions[modular_id].get(f"{spec}_instance", None) is None:
                self.functions[modular_id][f"{spec}_instance"] = instance
                self.functions[modular_id][f"{spec}_instance_type"] = instance_type
            else:
                self.print("Firest instance stays use new spec to get new instance")
                if modular_id in self.functions and self.functions[modular_id].get(f"{spec}_instance", None) is not None:
                    return self.functions[modular_id][f"{spec}_instance"]
                else:
                    raise ImportError(f"Module already known {modular_id} and not avalabel reload using other spec then {spec}")

        elif tools_class is not None:
            if modular_id not in self.functions:
                self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = tools_class
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

            try:
                if not hasattr(tools_class, 'tools'):
                    tools_class.tools = {"Version": tools_class.get_version, 'name': tools_class.name}
                for function_name in list(tools_class.tools.keys()):
                    t_function_name = function_name.lower()
                    if t_function_name != "all" and t_function_name != "name":
                        self.tb(function_name, mod_name=modular_id)(tools_class.tools.get(function_name))
                self.functions[modular_id][f"{spec}_instance_type"] += "/BC"
                if hasattr(tools_class, 'on_exit'):
                    if "on_exit" in self.functions[modular_id]:
                        self.functions[modular_id]["on_exit"].append(tools_class.on_exit)
                    else:
                        self.functions[modular_id]["on_exit"] = [tools_class.on_exit]
            except Exception as e:
                self.logger.error(f"Starting Module {modular_id} compatibility failed with : {e}")
                pass
        elif modular_id not in self.functions and tools_class is None:
            self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = instance
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

        else:
            raise ImportError(f"Modular {modular_id} is not a valid mod")
        on_start = self.functions[modular_id].get("on_start")
        if on_start is not None:
            i = 1
            for f in on_start:
                try:
                    f_, e = self.get_function((modular_id, f), state=True, specification=spec)
                    if e == 0:
                        self.logger.info(Style.GREY(f"Running On start {f} {i}/{len(on_start)}"))
                        if asyncio.iscoroutinefunction(f_):
                            self.print(f"Async on start is only in Tool claas supported for {modular_id}.{f}" if tools_class is None else f"initialization starting soon for {modular_id}.{f}")
                            self.run_bg_task_advanced(f_)
                        else:
                            o = f_()
                            if o is not None:
                                self.print(f"Function {modular_id} On start result: {o}")
                    else:
                        self.logger.warning(f"starting function not found {e}")
                except Exception as e:
                    self.logger.debug(Style.YELLOW(
                        Style.Bold(f"modular:{modular_id}.{f} on_start error {i}/{len(on_start)} -> {e}")))
                    self.debug_rains(e)
                finally:
                    i += 1
        return instance if tools_class is None else tools_class

    def save_initialized_module(self, tools_class, spec):
        tools_class.spec = spec
        live_tools_class = tools_class(app=self)
        return live_tools_class

    def mod_online(self, mod_name, installed=False):
        if installed and mod_name not in self.functions:
            self.save_load(mod_name)
        return mod_name in self.functions

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0, **kwargs):

        if as_str is None and isinstance(name, Enum):
            modular_id = str(name.NAME.value)
            function_id = str(name.value)
        elif as_str is None and isinstance(name, list):
            modular_id, function_id = name[0], name[1]
        else:
            modular_id, function_id = as_str

        self.logger.info(f"getting function : {specification}.{modular_id}.{function_id}")

        if modular_id not in self.functions:
            if r == 0:
                self.save_load(modular_id, spec=specification)
                return self.get_function(name=(modular_id, function_id),
                                         state=state,
                                         specification=specification,
                                         metadata=metadata,
                                         r=1)
            self.logger.warning(f"function modular not found {modular_id} 404")
            return "404", 404

        if function_id not in self.functions[modular_id]:
            self.logger.warning(f"function data not found {modular_id}.{function_id} 404")
            return "404", 404

        function_data = self.functions[modular_id][function_id]

        if isinstance(function_data, list):
            print(f"functions {function_id} : {function_data}")
            function_data = self.functions[modular_id][function_data[kwargs.get('i', -1)]]
            print(f"functions {modular_id} : {function_data}")
        function = function_data.get("func")
        params = function_data.get("params")

        state_ = function_data.get("state")
        if state_ is not None and state != state_:
            state = state_

        if function is None:
            self.logger.warning("No function found")
            return "404", 404

        if params is None:
            self.logger.warning("No function (params) found")
            return "404", 301

        if metadata and not state:
            self.logger.info("returning metadata stateless")
            return (function_data, function), 0

        if not state:  # mens a stateless function
            self.logger.info("returning stateless function")
            return function, 0

        instance = self.functions[modular_id].get(f"{specification}_instance")

        # instance_type = self.functions[modular_id].get(f"{specification}_instance_type", "functions/class")

        if params[0] == 'app':
            instance = get_app(from_=f"fuction {specification}.{modular_id}.{function_id}")

        if instance is None and self.alive:
            self.inplace_load_instance(modular_id, spec=specification)
            instance = self.functions[modular_id].get(f"{specification}_instance")

        if instance is None:
            self.logger.warning("No live Instance found")
            return "404", 400

        # if instance_type.endswith("/BC"):  # for backwards compatibility  functions/class/BC old modules
        #     # returning as stateless
        #     # return "422", -1
        #     self.logger.info(
        #         f"returning stateless function, cant find tools class for state handling found {instance_type}")
        #     if metadata:
        #         self.logger.info(f"returning metadata stateless")
        #         return (function_data, function), 0
        #     return function, 0

        self.logger.info("wrapping in higher_order_function")

        self.logger.info(f"returned fuction {specification}.{modular_id}.{function_id}")
        higher_order_function = partial(function, instance)

        if metadata:
            self.logger.info("returning metadata stateful")
            return (function_data, higher_order_function), 0

        self.logger.info("returning stateful function")
        return higher_order_function, 0

    def save_exit(self):
        self.logger.info(f"save exiting saving data to {self.config_fh.file_handler_filename} states of {self.debug=}")
        self.config_fh.add_to_save_file_handler(self.keys["debug"], str(self.debug))

    def init_mod(self, mod_name, spec='app'):
        """
        Initializes a module in a thread-safe manner by submitting the
        asynchronous initialization to the running event loop.
        """
        if '.' in mod_name:
            mod_name = mod_name.split('.')[0]
        self.run_bg_task(self.a_init_mod, mod_name, spec)
        # loop = self.loop_gard()
        # if loop:
        #     # Create a future to get the result from the coroutine
        #     future: Future = asyncio.run_coroutine_threadsafe(
        #         self.a_init_mod(mod_name, spec), loop
        #     )
        #     # Block until the result is available
        #     return future.result()
        # else:
        #     raise ValueError("Event loop is not running")
        #     # return self.loop_gard().run_until_complete(self.a_init_mod(mod_name, spec))

    def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
        """
        Runs a coroutine in the background without blocking the caller.

        This is the primary method for "fire-and-forget" async tasks. It schedules
        the coroutine to run on the application's main event loop.

        Args:
            task: The coroutine function to run.
            *args: Arguments to pass to the coroutine function.
            **kwargs: Keyword arguments to pass to the coroutine function.

        Returns:
            An asyncio.Task object representing the scheduled task, or None if
            the task could not be scheduled.
        """
        if not callable(task):
            self.logger.warning("Task passed to run_bg_task is not callable!")
            return None

        if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
            self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                                f"Use run_bg_task_advanced for synchronous functions.")
            # Fallback to advanced runner for convenience
            self.run_bg_task_advanced(task, *args, **kwargs)
            return None

        try:
            loop = self.loop_gard()
            if not loop.is_running():
                # If the main loop isn't running, we can't create a task on it.
                # This scenario is handled by run_bg_task_advanced.
                self.logger.info("Main event loop not running. Delegating to advanced background runner.")
                return self.run_bg_task_advanced(task, *args, **kwargs)

            # Create the coroutine if it's a function
            coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

            # Create a task on the running event loop
            bg_task = loop.create_task(coro)

            # Add a callback to log exceptions from the background task
            def _log_exception(the_task: asyncio.Task):
                if not the_task.cancelled() and the_task.exception():
                    self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                      exc_info=the_task.exception())

            bg_task.add_done_callback(_log_exception)
            self.bg_tasks.append(bg_task)
            return bg_task

        except Exception as e:
            self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
            return None

    def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
        """
        Runs a task in a separate, dedicated background thread with its own event loop.

        This is ideal for:
        1. Running an async task from a synchronous context.
        2. Launching a long-running, independent operation that should not
           interfere with the main application's event loop.

        Args:
            task: The function to run (can be sync or async).
            *args: Arguments for the task.
            **kwargs: Keyword arguments for the task.

        Returns:
            The threading.Thread object managing the background execution.
        """
        if not callable(task):
            self.logger.warning("Task for run_bg_task_advanced is not callable!")
            return None

        def thread_target():
            # Each thread gets its own event loop.
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

            try:
                # Prepare the coroutine we need to run
                if asyncio.iscoroutinefunction(task):
                    coro = task(*args, **kwargs)
                elif asyncio.iscoroutine(task):
                    # It's already a coroutine object
                    coro = task
                else:
                    # It's a synchronous function, run it in an executor
                    # to avoid blocking the new event loop.
                    coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

                # Run the coroutine to completion
                result = loop.run_until_complete(coro)
                self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
                if result is not None:
                    self.logger.debug(f"Task result: {str(result)[:100]}")

            except Exception as e:
                self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                                  exc_info=e)
            finally:
                # Cleanly shut down the event loop in this thread.
                try:
                    all_tasks = asyncio.all_tasks(loop=loop)
                    if all_tasks:
                        for t in all_tasks:
                            t.cancel()
                        loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
                finally:
                    loop.close()
                    asyncio.set_event_loop(None)

        # Create, start, and return the thread.
        # It's a daemon thread so it won't prevent the main app from exiting.
        t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
        self.bg_tasks.append(t)
        t.start()
        return t

    # Helper method to wait for background tasks to complete (optional)
    def wait_for_bg_tasks(self, timeout=None):
        """
        Wait for all background tasks to complete.

        Args:
            timeout: Maximum time to wait (in seconds) for all tasks to complete.
                     None means wait indefinitely.

        Returns:
            bool: True if all tasks completed, False if timeout occurred
        """
        active_tasks = [t for t in self.bg_tasks if t.is_alive()]

        for task in active_tasks:
            task.join(timeout=timeout)
            if task.is_alive():
                return False

        return True

    def __call__(self, *args, **kwargs):
        return self.run(*args, **kwargs)

    def run(self, *args, request=None, running_function_coro=None, **kwargs):
        """
        Run a function with support for SSE streaming in both
        threaded and non-threaded contexts.
        """
        if running_function_coro is None:
            mn, fn = args[0]
            if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
                kwargs["request"] = RequestData.from_dict(request)
                if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                    kwargs["request"].data = kwargs["request"].body = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                           []):
                    kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                    del kwargs['form_data']
            else:
                params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
                # auto pars data and form_data to kwargs by key
                do = False
                data = {}
                if 'data' in kwargs and 'data' not in params:
                    do = True
                    data = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in params:
                    do = True
                    data = kwargs['form_data']
                    del kwargs['form_data']
                if do:
                    for k in params:
                        if k in data:
                            kwargs[k] = data[k]
                            del data[k]

        # Create the coroutine
        coro = running_function_coro or self.a_run_any(*args, **kwargs)

        # Get or create an event loop
        try:
            loop = asyncio.get_event_loop()
            is_running = loop.is_running()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            is_running = False

        # If the loop is already running, run in a separate thread
        if is_running:
            # Create thread pool executor as needed
            if not hasattr(self.__class__, '_executor'):
                self.__class__._executor = ThreadPoolExecutor(max_workers=4)

            def run_in_new_thread():
                # Set up a new loop in this thread
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)

                try:
                    # Run the coroutine
                    return new_loop.run_until_complete(coro)
                finally:
                    new_loop.close()

            # Run in thread and get result
            thread_result = self.__class__._executor.submit(run_in_new_thread).result()

            # Handle streaming results from thread
            if isinstance(thread_result, dict) and thread_result.get("is_stream"):
                # Create a new SSE stream in the main thread
                async def stream_from_function():
                    # Re-run the function with direct async access
                    stream_result = await self.a_run_any(*args, **kwargs)

                    if (isinstance(stream_result, Result) and
                        getattr(stream_result.result, 'data_type', None) == "stream"):
                        # Get and forward data from the original generator
                        original_gen = stream_result.result.data.get("generator")
                        if inspect.isasyncgen(original_gen):
                            async for item in original_gen:
                                yield item

                # Return a new streaming Result
                return Result.stream(
                    stream_generator=stream_from_function(),
                    headers=thread_result.get("headers", {})
                )

            result = thread_result
        else:
            # Direct execution when loop is not running
            result = loop.run_until_complete(coro)

        # Process the final result
        if isinstance(result, Result):
            if 'debug' in self.id:
                result.print()
            if getattr(result.result, 'data_type', None) == "stream":
                return result
            return result.to_api_result().model_dump(mode='json')

        return result

    def loop_gard(self):
        if self.loop is None:
            self._start_event_loop()
            self.loop = asyncio.get_event_loop()
        if self.loop.is_closed():
            self.loop = asyncio.get_event_loop()
        return self.loop

    async def a_init_mod(self, mod_name, spec='app'):
        mod = self.save_load(mod_name, spec=spec)
        if hasattr(mod, "__initobj") and not mod.async_initialized:
            await mod
        return mod


    def load_mod(self, mod_name: str, mlm='I', **kwargs):

        action_list_helper = ['I (inplace load dill on error python)',
                              # 'C (coppy py file to runtime dir)',
                              # 'S (save py file to dill)',
                              # 'CS (coppy and save py file)',
                              # 'D (development mode, inplace load py file)'
                              ]
        action_list = {"I": lambda: self.inplace_load_instance(mod_name, **kwargs),
                       "C": lambda: self._copy_load(mod_name, **kwargs)
                       }

        try:
            if mlm in action_list:

                return action_list.get(mlm)()
            else:
                self.logger.critical(
                    f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
                raise ValueError(f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
        except ValueError as e:
            self.logger.warning(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except ImportError as e:
            self.logger.error(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except Exception as e:
            self.logger.critical(Style.RED(f"Error Loading Module '{mod_name}', with critical error :{e}"))
            print(Style.RED(f"Error Loading Module '{mod_name}'"))
            self.debug_rains(e)

        return Result.default_internal_error(info="info's in logs.")

    async def load_external_mods(self):
        for mod_path in os.getenv("EXTERNAL_PATH_RUNNABLE", '').split(','):
            if mod_path:
                await self.load_all_mods_in_file(mod_path)

    async def load_all_mods_in_file(self, working_dir="mods"):
        print(f"LOADING ALL MODS FROM FOLDER : {working_dir}")
        t0 = time.perf_counter()
        # Get the list of all modules
        module_list = self.get_all_mods(working_dir)
        open_modules = self.functions.keys()
        start_len = len(open_modules)

        for om in open_modules:
            if om in module_list:
                module_list.remove(om)

        tasks: set[Task] = set()

        _ = {tasks.add(asyncio.create_task(asyncio.to_thread(self.save_load, mod, 'app'))) for mod in module_list}
        for t in asyncio.as_completed(tasks):
            try:
                result = await t
                if hasattr(result, 'Name'):
                    self.print('Opened :', result.Name)
                elif hasattr(result, 'name'):
                    if hasattr(result, 'async_initialized'):
                        if not result.async_initialized:
                            async def _():
                                try:
                                    if asyncio.iscoroutine(result):
                                        await result
                                    if hasattr(result, 'Name'):
                                        self.print('Opened :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Opened :', result.name)
                                except Exception as e:
                                    self.debug_rains(e)
                                    if hasattr(result, 'Name'):
                                        self.print('Error opening :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Error opening :', result.name)
                            asyncio.create_task(_())
                        else:
                            self.print('Opened :', result.name)
                else:
                    if result:
                        self.print('Opened :', result)
            except Exception as e:
                self.logger.error(Style.RED(f"An Error occurred while opening all modules error: {str(e)}"))
                self.debug_rains(e)
        opened = len(self.functions.keys()) - start_len

        self.logger.info(f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s")
        return f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s"

    def get_all_mods(self, working_dir="mods", path_to="./runtime", use_wd=True):
        self.logger.info(f"collating all mods in working directory {working_dir}")

        pr = "_dev" if self.dev_modi else ""
        if working_dir == "mods" and use_wd:
            working_dir = f"{self.start_dir}/mods{pr}"
        elif use_wd:
            pass
        else:
            w_dir = self.id.replace(".", "_")
            working_dir = f"{path_to}/{w_dir}/mod_lib{pr}/"
        res = os.listdir(working_dir)

        self.logger.info(f"found : {len(res)} files")

        def do_helper(_mod):
            if "mainTool" in _mod:
                return False
            # if not _mod.endswith(".py"):
            #     return False
            if _mod.startswith("__"):
                return False
            if _mod.startswith("."):
                return False
            return not _mod.startswith("test_")

        def r_endings(word: str):
            if word.endswith(".py"):
                return word[:-3]
            return word

        mods_list = list(map(r_endings, filter(do_helper, res)))

        self.logger.info(f"found : {len(mods_list)} Modules")
        return mods_list

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    def remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return

        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    self.exit_tasks.append(instance.on_exit)
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1

        for j, f in enumerate(on_exit):
            try:
                f_, e = self.get_function((mod_name, f), state=True, specification=spec, i=j)
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        self.exit_tasks.append(f_)
                        o = None
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))

                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return
        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    await instance.on_exit()
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1
        for f in on_exit:
            try:
                e = 1
                if isinstance(f, str):
                    f_, e = self.get_function((mod_name, f), state=True, specification=spec)
                elif isinstance(f, Callable):
                    f_, e, f  = f, 0, f.__name__
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        o = await f_()
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))
                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    def exit(self, remove_all=True):
        if not self.alive:
            return
        if self.args_sto.debug:
            self.hide_console()
        self.disconnect()
        if remove_all:
            self.remove_all_modules()
        self.logger.info("Exiting ToolBox interface")
        self.alive = False
        self.called_exit = True, time.time()
        self.save_exit()
        if hasattr(self, 'root_blob_storage') and self.root_blob_storage:
            self.root_blob_storage.exit()
        try:
            self.config_fh.save_file_handler()
        except SystemExit:
            print("If u ar testing this is fine else ...")

        if hasattr(self, 'daemon_app'):
            import threading

            for thread in threading.enumerate()[::-1]:
                if thread.name == "MainThread":
                    continue
                try:
                    with Spinner(f"closing Thread {thread.name:^50}|", symbols="s", count_down=True,
                                 time_in_s=0.751 if not self.debug else 0.6):
                        thread.join(timeout=0.751 if not self.debug else 0.6)
                except TimeoutError as e:
                    self.logger.error(f"Timeout error on exit {thread.name} {str(e)}")
                    print(str(e), f"Timeout {thread.name}")
                except KeyboardInterrupt:
                    print("Unsave Exit")
                    break
        if hasattr(self, 'loop') and self.loop is not None:
            with Spinner("closing Event loop:", symbols="+"):
                self.loop.stop()

    async def a_exit(self):

        import inspect
        self.sprint(f"exit requested from: {inspect.stack()[1].filename}::{inspect.stack()[1].lineno} function: {inspect.stack()[1].function}")

        # Cleanup session before removing modules
        try:
            if hasattr(self, 'session') and self.session is not None:
                await self.session.cleanup()
        except Exception as e:
            self.logger.debug(f"Session cleanup error (ignored): {e}")

        await self.a_remove_all_modules(delete=True)
        results = await asyncio.gather(
            *[asyncio.create_task(f()) for f in self.exit_tasks if asyncio.iscoroutinefunction(f)])
        for result in results:
            self.print(f"Function On Exit result: {result}")
        self.exit(remove_all=False)

    def save_load(self, modname, spec='app'):
        self.logger.debug(f"Save load module {modname}")
        if not modname:
            self.logger.warning("no filename specified")
            return False
        try:
            return self.load_mod(modname, spec=spec)
        except ModuleNotFoundError as e:
            self.logger.error(Style.RED(f"Module {modname} not found"))
            self.debug_rains(e)

        return False

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """
        if isinstance(name, tuple):
            return self._get_function(None, as_str=name, **kwargs)
        else:
            return self._get_function(name, **kwargs)

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if tb_run_with_specification == 'ws_internal':
            modular_name = modular_name.split('/')[0]
            if not self.mod_online(modular_name, installed=True):
                self.get_mod(modular_name)
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        await handler_func(self, **kwargs)
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 404:
            mod = self.get_mod(modular_name)
            if hasattr(mod, "async_initialized") and not mod.async_initialized:
                await mod
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 404:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == 300:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            return await self.a_fuction_runner(function, function_data, args, kwargs, t0)
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)


    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        if tb_run_with_specification == 'ws_internal':
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        return self.loop.run_until_complete(handler_func(self, **kwargs))
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 1 or error_code == 3 or error_code == 400:
            self.get_mod(modular_name)
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 2:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == -1:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            try:
                return asyncio.run(self.a_fuction_runner(function, function_data, args, kwargs, t0))
            except RuntimeError:
                try:
                    return self.loop.run_until_complete(self.a_fuction_runner(function, function_data, args, kwargs, t0))
                except RuntimeError:
                    pass
            raise ValueError(f"Fuction {function_name} is Async use a_run_any")
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)

    def run_a_from_sync(self, function, *args, **kwargs):
        # Initialize self.loop if not already set.
        if self.loop is None:
            try:
                self.loop = asyncio.get_running_loop()
            except RuntimeError:
                self.loop = asyncio.new_event_loop()

        # If the loop is running, offload the coroutine to a new thread.
        if self.loop.is_running():
            result_future = Future()

            def run_in_new_loop():
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)
                try:
                    result = new_loop.run_until_complete(function(*args, **kwargs))
                    result_future.set_result(result)
                except Exception as e:
                    result_future.set_exception(e)
                finally:
                    new_loop.close()

            thread = threading.Thread(target=run_in_new_loop)
            thread.start()
            thread.join()  # Block until the thread completes.
            return result_future.result()
        else:
            # If the loop is not running, schedule and run the coroutine directly.
            future = self.loop.create_task(function(*args, **kwargs))
            return self.loop.run_until_complete(future)

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = function()
            elif len(parameters) == len(args) + if_self_state:
                res = function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = function(**kwargs)
            else:
                res = function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)
            self.print(f"! Function ERROR: in {modular_name}.{function_name} ")



        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = await function()
            elif len(parameters) == len(args) + if_self_state:
                res = await function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = await function(**kwargs)
            else:
                res = await function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)

        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None,
                       args_=None,
                       kwargs_=None, method="GET",
                       *args, **kwargs):
        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        modular_name = mod_function_name
        function_name = function_name

        if isinstance(mod_function_name, str) and isinstance(function_name, str):
            mod_function_name = (mod_function_name, function_name)

        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value

        self.logger.info(f"getting function : {modular_name}.{function_name} from http {self.session.base}")
        r = await self.session.fetch(f"/api/{modular_name}/{function_name}{'?' + args_ if args_ is not None else ''}",
                                     data=kwargs, method=method)
        try:
            if not r:
                print("§ Session server Offline!", self.session.base)
                return Result.default_internal_error(info="Session fetch failed").as_dict()

            content_type = r.headers.get('Content-Type', '').lower()

            if 'application/json' in content_type:
                try:
                    return r.json()
                except Exception as e:
                    print(f"⚠ JSON decode error: {e}")
                    # Fallback to text if JSON decoding fails
                    text = r.text
            else:
                text = r.text

            if isinstance(text, Callable):
                if asyncio.iscoroutinefunction(text):
                    text = await text()
                else:
                    text = text()

            # Attempt YAML
            if 'yaml' in content_type or text.strip().startswith('---'):
                try:
                    import yaml
                    return yaml.safe_load(text)
                except Exception as e:
                    print(f"⚠ YAML decode error: {e}")

            # Attempt XML
            if 'xml' in content_type or text.strip().startswith('<?xml'):
                try:
                    import xmltodict
                    return xmltodict.parse(text)
                except Exception as e:
                    print(f"⚠ XML decode error: {e}")

            # Fallback: return plain text
            return Result.default_internal_error(data={'raw_text': text, 'content_type': content_type}).as_dict()

        except Exception as e:
            print("❌ Fatal error during API call:", e)
            self.debug_rains(e)
            return Result.default_internal_error(str(e)).as_dict()

    def run_local(self, *args, **kwargs):
        return self.run_any(*args, **kwargs)

    async def a_run_local(self, *args, **kwargs):
        return await self.a_run_any(*args, **kwargs)

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = self.run_function(mod_function_name,
                                        tb_run_function_with_state=tb_run_function_with_state,
                                        tb_run_with_specification=tb_run_with_specification,
                                        args_=args, kwargs_=kwargs).as_result()
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.log(show_data=False)

        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = await self.a_run_function(mod_function_name,
                                                tb_run_function_with_state=tb_run_function_with_state,
                                                tb_run_with_specification=tb_run_with_specification,
                                                args_=args, kwargs_=kwargs)
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.print()
            res.log(show_data=False) if isinstance(res, Result) else self.logger.debug(res)
        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res


    def web_context(self):
        if self._web_context is None:
            try:
                self._web_context = open("./dist/helper.html", encoding="utf-8").read()
            except Exception as e:
                self.logger.error(f"Could not load web context: {e}")
                self._web_context = "<div><h1>Web Context not found</h1></div>"
        return self._web_context

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        if spec != "app":
            self.print(f"Getting Module {name} spec: {spec}")
        if name not in self.functions:
            mod = self.save_load(name, spec=spec)
            if mod is False or (isinstance(mod, Result) and mod.is_error()):
                self.logger.warning(f"Could not find {name} in {list(self.functions.keys())}")
                raise ValueError(f"Could not find {name} in {list(self.functions.keys())} pleas install the module, or its posibly broken use --debug for infos")
        # private = self.functions[name].get(f"{spec}_private")
        # if private is not None:
        #     if private and spec != 'app':
        #         raise ValueError("Module is private")
        if name not in self.functions:
            self.logger.warning(f"Module '{name}' is not found")
            return None
        instance = self.functions[name].get(f"{spec}_instance")
        if instance is None:
            return self.load_mod(name, spec=spec)
        return self.functions[name].get(f"{spec}_instance")

    def print(self, text="", *args, **kwargs):
        # self.logger.info(f"Output : {text}")
        if 'live' in self.id:
            return

        flush = kwargs.pop('flush', True)
        if self.sprint(None):
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        print(text, *args, **kwargs, flush=flush)

    def sprint(self, text="", show_system=True, *args, **kwargs):
        if text is None:
            return True
        if 'live' in self.id:
            return
        flush = kwargs.pop('flush', True)
        # self.logger.info(f"Output : {text}")
        if show_system:
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        if isinstance(text, str) and kwargs == {} and text:
            stram_print(text + ' '.join(args))
            print()
        else:
            print(text, *args, **kwargs, flush=flush)

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        self.remove_mod(mod_name, delete=True)
        if mod_name not in self.modules:
            self.logger.warning(f"Module '{mod_name}' is not found")
            return
        if hasattr(self.modules[mod_name], 'reload_save') and self.modules[mod_name].reload_save:
            def reexecute_module_code(x):
                return x
        else:
            def reexecute_module_code(module_name):
                if isinstance(module_name, str):
                    module = import_module(module_name)
                else:
                    module = module_name
                # Get the source code of the module
                try:
                    source = inspect.getsource(module)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    return module
                # Compile the source code
                try:
                    code = compile(source, module.__file__, 'exec')
                    # Execute the code in the module's namespace
                    exec(code, module.__dict__)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    pass
                return module

        if not is_file:
            mods = self.get_all_mods("./mods/" + mod_name)
            def recursive_reload(package_name):
                package = import_module(package_name)

                # First, reload all submodules
                if hasattr(package, '__path__'):
                    for _finder, name, _ispkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
                        try:
                            mod = import_module(name)
                            reexecute_module_code(mod)
                            reload(mod)
                        except Exception as e:
                            print(f"Error reloading module {name}: {e}")
                            break

                # Finally, reload the package itself
                reexecute_module_code(package)
                reload(package)

            for mod in mods:
                if mod.endswith(".txt") or mod.endswith(".yaml"):
                    continue
                try:
                    recursive_reload(loc + mod_name + '.' + mod)
                    self.print(f"Reloaded {mod_name}.{mod}")
                except ImportError:
                    self.print(f"Could not load {mod_name}.{mod}")
        reexecute_module_code(self.modules[mod_name])
        if mod_name in self.functions:
            if "on_exit" in self.functions[mod_name]:
                self.functions[mod_name]["on_exit"] = []
            if "on_start" in self.functions[mod_name]:
                self.functions[mod_name]["on_start"] = []
        self.inplace_load_instance(mod_name, spec=spec, mfo=reload(self.modules[mod_name]) if mod_name in self.modules else None)

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None, on_reload=None):
        if path_name is None:
            path_name = mod_name
        is_file = os.path.isfile(self.start_dir + '/mods/' + path_name + '.py')
        import watchfiles
        def helper():
            paths = f'mods/{path_name}' + ('.py' if is_file else '')
            self.logger.info(f'Watching Path: {paths}')
            try:
                for changes in watchfiles.watch(paths):
                    if not changes:
                        continue
                    self.reload_mod(mod_name, spec, is_file, loc)
                    if on_reload:
                        on_reload()
            except FileNotFoundError:
                self.logger.warning(f"Path {paths} not found")

        if not use_thread:
            helper()
        else:
            threading.Thread(target=helper, daemon=True).start()

    def _register_function(self, module_name, func_name, data):
        if module_name not in self.functions:
            self.functions[module_name] = {}
        if func_name in self.functions[module_name]:
            self.print(f"Overriding function {func_name} from {module_name}", end="\r")
            self.functions[module_name][func_name] = data
        else:
            self.functions[module_name][func_name] = data

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial: bool=False,
                          exit_f: bool=False,
                          test: bool=True,
                          samples:list[dict[str, Any]] | None=None,
                          state:bool | None=None,
                          pre_compute:Callable | None=None,
                          post_compute:Callable[[], Result] | None=None,
                          api_methods:list[str] | None=None,
                          memory_cache: bool=False,
                          file_cache: bool=False,
                          request_as_kwarg: bool=False,
                          row: bool=False,
                          memory_cache_max_size:int=100,
                          memory_cache_ttl:int=300,
                          websocket_handler: str | None = None,
                          ):

        if isinstance(type_, Enum):
            type_ = type_.value

        if memory_cache and file_cache:
            raise ValueError("Don't use both cash at the same time for the same fuction")

        use_cache = memory_cache or file_cache
        cache = {}
        if file_cache:
            cache = FileCache(folder=self.data_dir + f'\\cache\\{mod_name}\\',
                              filename=self.data_dir + f'\\cache\\{mod_name}\\{name}cache.db')
        if memory_cache:
            cache = MemoryCache(maxsize=memory_cache_max_size, ttl=memory_cache_ttl)

        version = self.version if version is None else self.version + ':' + version

        def a_additional_process(func):

            async def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = await pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = await func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = await post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            async def wrapper(*args, **kwargs):

                if not use_cache:
                    return await executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = await executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def additional_process(func):

            def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            def wrapper(*args, **kwargs):

                if not use_cache:
                    return executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def decorator(func):
            sig = signature(func)
            params = list(sig.parameters)
            module_name = mod_name if mod_name else func.__module__.split('.')[-1]
            func_name = name if name else func.__name__
            if func_name == 'on_start':
                func_name = 'on_startup'
            if func_name == 'on_exit':
                func_name = 'on_close'
            if api or pre_compute is not None or post_compute is not None or memory_cache or file_cache:
                if asyncio.iscoroutinefunction(func):
                    func = a_additional_process(func)
                else:
                    func = additional_process(func)
            if api and str(sig.return_annotation) == 'Result':
                raise ValueError(f"Fuction {module_name}.{func_name} registered as "
                                 f"Api fuction but uses {str(sig.return_annotation)}\n"
                                 f"Please change the sig from ..)-> Result to ..)-> ApiResult")
            data = {
                "type": type_,
                "module_name": module_name,
                "func_name": func_name,
                "level": level,
                "restrict_in_virtual_mode": restrict_in_virtual_mode,
                "func": func,
                "api": api,
                "helper": helper,
                "version": version,
                "initial": initial,
                "exit_f": exit_f,
                "api_methods": api_methods if api_methods is not None else ["AUTO"],
                "__module__": func.__module__,
                "signature": sig,
                "params": params,
                "row": row,
                "state": (
                    False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
                "do_test": test,
                "samples": samples,
                "request_as_kwarg": request_as_kwarg,

            }

            if websocket_handler:
                # Die dekorierte Funktion sollte ein Dict mit den Handlern zurückgeben
                try:
                    handler_config = func(self)  # Rufe die Funktion auf, um die Konfiguration zu erhalten
                    if not isinstance(handler_config, dict):
                        raise TypeError(
                            f"WebSocket handler function '{func.__name__}' must return a dictionary of handlers.")

                    # Handler-Identifikator, z.B. "ChatModule/room_chat"
                    handler_id = f"{module_name}/{websocket_handler}"
                    self.websocket_handlers[handler_id] = {}

                    for event_name, handler_func in handler_config.items():
                        if event_name in ["on_connect", "on_message", "on_disconnect"] and callable(handler_func):
                            self.websocket_handlers[handler_id][event_name] = handler_func
                        else:
                            self.logger.warning(f"Invalid WebSocket handler event '{event_name}' in '{handler_id}'.")

                    self.logger.info(f"Registered WebSocket handlers for '{handler_id}'.")

                except Exception as e:
                    self.logger.error(f"Failed to register WebSocket handlers for '{func.__name__}': {e}",
                                      exc_info=True)
            else:
                self._register_function(module_name, func_name, data)

            if exit_f:
                if "on_exit" not in self.functions[module_name]:
                    self.functions[module_name]["on_exit"] = []
                self.functions[module_name]["on_exit"].append(func_name)
            if initial:
                if "on_start" not in self.functions[module_name]:
                    self.functions[module_name]["on_start"] = []
                self.functions[module_name]["on_start"].append(func_name)

            return func

        decorator.tb_init = True

        return decorator

    def export(self, *args, **kwargs):
        return self.tb(*args, **kwargs)

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str | None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           request_as_kwarg: bool = False,
           row: bool = False,
           state: bool | None = None,
           level: int = -1,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
        websocket_handler (str, optional): The name of the websocket handler to use.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      request_as_kwarg=request_as_kwarg,
                                      row=row,
                                      api_methods=api_methods,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl,
                                      websocket_handler=websocket_handler,
                                      )

    def save_autocompletion_dict(self):
        autocompletion_dict = {}
        for module_name, _module in self.functions.items():
            data = {}
            for function_name, function_data in self.functions[module_name].items():
                if not isinstance(function_data, dict):
                    continue
                data[function_name] = {arg: None for arg in
                                       function_data.get("params", [])}
                if len(data[function_name].keys()) == 0:
                    data[function_name] = None
            autocompletion_dict[module_name] = data if len(data.keys()) > 0 else None
        self.config_fh.add_to_save_file_handler("auto~~~~~~", str(autocompletion_dict))

    def get_autocompletion_dict(self):
        return self.config_fh.get_file_handler("auto~~~~~~")

    def save_registry_as_enums(self, directory: str, filename: str):
        # Ordner erstellen, falls nicht vorhanden
        if not os.path.exists(directory):
            os.makedirs(directory)

        # Dateipfad vorbereiten
        filepath = os.path.join(directory, filename)

        # Enum-Klassen als Strings generieren
        enum_classes = [f'"""Automatic generated by ToolBox v = {self.version}"""'
                        f'\nfrom enum import Enum\nfrom dataclasses import dataclass'
                        f'\n\n\n']
        for module, functions in self.functions.items():
            if module.startswith("APP_INSTANCE"):
                continue
            class_name = module
            enum_members = "\n    ".join(
                [
                    f"{func_name.upper().replace('-', '')}"
                    f" = '{func_name}' "
                    f"# Input: ({fuction_data['params'] if isinstance(fuction_data, dict) else ''}),"
                    f" Output: {fuction_data['signature'].return_annotation if isinstance(fuction_data, dict) else 'None'}"
                    for func_name, fuction_data in functions.items()])
            enum_class = (f'@dataclass\nclass {class_name.upper().replace(".", "_").replace("-", "")}(Enum):'
                          f"\n    NAME = '{class_name}'\n    {enum_members}")
            enum_classes.append(enum_class)

        # Enums in die Datei schreiben
        data = "\n\n\n".join(enum_classes)
        if len(data) < 12:
            raise ValueError(
                "Invalid Enums Loosing content pleas delete it ur self in the (utils/system/all_functions_enums.py) or add mor new stuff :}")
        with open(filepath, 'w') as file:
            file.write(data)

        print(Style.Bold(Style.BLUE(f"Enums gespeichert in {filepath}")))


    # WS logic

    def _set_rust_ws_bridge(self, bridge_object: Any):
        """
        Diese Methode wird von Rust aufgerufen, um die Kommunikationsbrücke zu setzen.
        Sie darf NICHT manuell von Python aus aufgerufen werden.
        """
        self.print(f"Rust WebSocket bridge has been set for instance {self.id}.")
        self._rust_ws_bridge = bridge_object

    async def ws_send(self, conn_id: str, payload: dict):
        """
        Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

        Args:
            conn_id: Die eindeutige ID der Zielverbindung.
            payload: Ein Dictionary, das als JSON gesendet wird.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
            await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
        except Exception as e:
            self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

    async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
        """
        Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

        Args:
            channel_id: Der Kanal, an den gesendet werden soll.
            payload: Ein Dictionary, das als JSON gesendet wird.
            source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Broadcast-Methode auf
            await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
        except Exception as e:
            self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)
disconnect(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
244
245
246
@staticmethod
def disconnect(*args, **kwargs):
    """proxi attr"""
exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
232
233
234
@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""
get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/toolbox.py
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """
    if isinstance(name, tuple):
        return self._get_function(None, as_str=name, **kwargs)
    else:
        return self._get_function(name, **kwargs)
hide_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
236
237
238
@staticmethod
def hide_console(*args, **kwargs):
    """proxi attr"""
init_mod(mod_name, spec='app')

Initializes a module in a thread-safe manner by submitting the asynchronous initialization to the running event loop.

Source code in toolboxv2/utils/toolbox.py
625
626
627
628
629
630
631
632
def init_mod(self, mod_name, spec='app'):
    """
    Initializes a module in a thread-safe manner by submitting the
    asynchronous initialization to the running event loop.
    """
    if '.' in mod_name:
        mod_name = mod_name.split('.')[0]
    self.run_bg_task(self.a_init_mod, mod_name, spec)
run(*args, request=None, running_function_coro=None, **kwargs)

Run a function with support for SSE streaming in both threaded and non-threaded contexts.

Source code in toolboxv2/utils/toolbox.py
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
def run(self, *args, request=None, running_function_coro=None, **kwargs):
    """
    Run a function with support for SSE streaming in both
    threaded and non-threaded contexts.
    """
    if running_function_coro is None:
        mn, fn = args[0]
        if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
            kwargs["request"] = RequestData.from_dict(request)
            if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                kwargs["request"].data = kwargs["request"].body = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                       []):
                kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                del kwargs['form_data']
        else:
            params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
            # auto pars data and form_data to kwargs by key
            do = False
            data = {}
            if 'data' in kwargs and 'data' not in params:
                do = True
                data = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in params:
                do = True
                data = kwargs['form_data']
                del kwargs['form_data']
            if do:
                for k in params:
                    if k in data:
                        kwargs[k] = data[k]
                        del data[k]

    # Create the coroutine
    coro = running_function_coro or self.a_run_any(*args, **kwargs)

    # Get or create an event loop
    try:
        loop = asyncio.get_event_loop()
        is_running = loop.is_running()
    except RuntimeError:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        is_running = False

    # If the loop is already running, run in a separate thread
    if is_running:
        # Create thread pool executor as needed
        if not hasattr(self.__class__, '_executor'):
            self.__class__._executor = ThreadPoolExecutor(max_workers=4)

        def run_in_new_thread():
            # Set up a new loop in this thread
            new_loop = asyncio.new_event_loop()
            asyncio.set_event_loop(new_loop)

            try:
                # Run the coroutine
                return new_loop.run_until_complete(coro)
            finally:
                new_loop.close()

        # Run in thread and get result
        thread_result = self.__class__._executor.submit(run_in_new_thread).result()

        # Handle streaming results from thread
        if isinstance(thread_result, dict) and thread_result.get("is_stream"):
            # Create a new SSE stream in the main thread
            async def stream_from_function():
                # Re-run the function with direct async access
                stream_result = await self.a_run_any(*args, **kwargs)

                if (isinstance(stream_result, Result) and
                    getattr(stream_result.result, 'data_type', None) == "stream"):
                    # Get and forward data from the original generator
                    original_gen = stream_result.result.data.get("generator")
                    if inspect.isasyncgen(original_gen):
                        async for item in original_gen:
                            yield item

            # Return a new streaming Result
            return Result.stream(
                stream_generator=stream_from_function(),
                headers=thread_result.get("headers", {})
            )

        result = thread_result
    else:
        # Direct execution when loop is not running
        result = loop.run_until_complete(coro)

    # Process the final result
    if isinstance(result, Result):
        if 'debug' in self.id:
            result.print()
        if getattr(result.result, 'data_type', None) == "stream":
            return result
        return result.to_api_result().model_dump(mode='json')

    return result
run_bg_task(task, *args, **kwargs)

Runs a coroutine in the background without blocking the caller.

This is the primary method for "fire-and-forget" async tasks. It schedules the coroutine to run on the application's main event loop.

Parameters:

Name Type Description Default
task Callable

The coroutine function to run.

required
*args

Arguments to pass to the coroutine function.

()
**kwargs

Keyword arguments to pass to the coroutine function.

{}

Returns:

Type Description
Task | None

An asyncio.Task object representing the scheduled task, or None if

Task | None

the task could not be scheduled.

Source code in toolboxv2/utils/toolbox.py
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
    """
    Runs a coroutine in the background without blocking the caller.

    This is the primary method for "fire-and-forget" async tasks. It schedules
    the coroutine to run on the application's main event loop.

    Args:
        task: The coroutine function to run.
        *args: Arguments to pass to the coroutine function.
        **kwargs: Keyword arguments to pass to the coroutine function.

    Returns:
        An asyncio.Task object representing the scheduled task, or None if
        the task could not be scheduled.
    """
    if not callable(task):
        self.logger.warning("Task passed to run_bg_task is not callable!")
        return None

    if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
        self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                            f"Use run_bg_task_advanced for synchronous functions.")
        # Fallback to advanced runner for convenience
        self.run_bg_task_advanced(task, *args, **kwargs)
        return None

    try:
        loop = self.loop_gard()
        if not loop.is_running():
            # If the main loop isn't running, we can't create a task on it.
            # This scenario is handled by run_bg_task_advanced.
            self.logger.info("Main event loop not running. Delegating to advanced background runner.")
            return self.run_bg_task_advanced(task, *args, **kwargs)

        # Create the coroutine if it's a function
        coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

        # Create a task on the running event loop
        bg_task = loop.create_task(coro)

        # Add a callback to log exceptions from the background task
        def _log_exception(the_task: asyncio.Task):
            if not the_task.cancelled() and the_task.exception():
                self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                  exc_info=the_task.exception())

        bg_task.add_done_callback(_log_exception)
        self.bg_tasks.append(bg_task)
        return bg_task

    except Exception as e:
        self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
        return None
run_bg_task_advanced(task, *args, **kwargs)

Runs a task in a separate, dedicated background thread with its own event loop.

This is ideal for: 1. Running an async task from a synchronous context. 2. Launching a long-running, independent operation that should not interfere with the main application's event loop.

Parameters:

Name Type Description Default
task Callable

The function to run (can be sync or async).

required
*args

Arguments for the task.

()
**kwargs

Keyword arguments for the task.

{}

Returns:

Type Description
Thread

The threading.Thread object managing the background execution.

Source code in toolboxv2/utils/toolbox.py
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
    """
    Runs a task in a separate, dedicated background thread with its own event loop.

    This is ideal for:
    1. Running an async task from a synchronous context.
    2. Launching a long-running, independent operation that should not
       interfere with the main application's event loop.

    Args:
        task: The function to run (can be sync or async).
        *args: Arguments for the task.
        **kwargs: Keyword arguments for the task.

    Returns:
        The threading.Thread object managing the background execution.
    """
    if not callable(task):
        self.logger.warning("Task for run_bg_task_advanced is not callable!")
        return None

    def thread_target():
        # Each thread gets its own event loop.
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

        try:
            # Prepare the coroutine we need to run
            if asyncio.iscoroutinefunction(task):
                coro = task(*args, **kwargs)
            elif asyncio.iscoroutine(task):
                # It's already a coroutine object
                coro = task
            else:
                # It's a synchronous function, run it in an executor
                # to avoid blocking the new event loop.
                coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

            # Run the coroutine to completion
            result = loop.run_until_complete(coro)
            self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
            if result is not None:
                self.logger.debug(f"Task result: {str(result)[:100]}")

        except Exception as e:
            self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                              exc_info=e)
        finally:
            # Cleanly shut down the event loop in this thread.
            try:
                all_tasks = asyncio.all_tasks(loop=loop)
                if all_tasks:
                    for t in all_tasks:
                        t.cancel()
                    loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
            finally:
                loop.close()
                asyncio.set_event_loop(None)

    # Create, start, and return the thread.
    # It's a daemon thread so it won't prevent the main app from exiting.
    t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
    self.bg_tasks.append(t)
    t.start()
    return t
show_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
240
241
242
@staticmethod
def show_console(*args, **kwargs):
    """proxi attr"""
tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, request_as_kwarg=False, row=False, state=None, level=-1, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name Type Description Default
name str

The name to register the function under. Defaults to the function's own name.

None
mod_name str

The name of the module the function belongs to.

''
helper str

A helper string providing additional information about the function.

''
version str or None

The version of the function or module.

None
test bool

Flag to indicate if the function is for testing purposes.

True
restrict_in_virtual_mode bool

Flag to restrict the function in virtual mode.

False
api bool

Flag to indicate if the function is part of an API.

False
initial bool

Flag to indicate if the function should be executed at initialization.

False
exit_f bool

Flag to indicate if the function should be executed at exit.

False
test_only bool

Flag to indicate if the function should only be used for testing.

False
memory_cache bool

Flag to enable memory caching for the function.

False
request_as_kwarg bool

Flag to get request if the fuction is calld from api.

False
file_cache bool

Flag to enable file caching for the function.

False
row bool

rather to auto wrap the result in Result type default False means no row data aka result type

False
state bool or None

Flag to indicate if the function maintains state.

None
level int

The level of the function, used for prioritization or categorization.

-1
memory_cache_max_size int

Maximum size of the memory cache.

100
memory_cache_ttl int

Time-to-live for the memory cache entries.

300
samples list or dict or None

Samples or examples of function usage.

None
interface str

The interface type for the function.

None
pre_compute callable

A function to be called before the main function.

None
post_compute callable

A function to be called after the main function.

None
api_methods list[str]

default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

None
websocket_handler str

The name of the websocket handler to use.

None

Returns:

Name Type Description
function

The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/toolbox.py
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str | None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       request_as_kwarg: bool = False,
       row: bool = False,
       state: bool | None = None,
       level: int = -1,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
    websocket_handler (str, optional): The name of the websocket handler to use.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  request_as_kwarg=request_as_kwarg,
                                  row=row,
                                  api_methods=api_methods,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl,
                                  websocket_handler=websocket_handler,
                                  )
wait_for_bg_tasks(timeout=None)

Wait for all background tasks to complete.

Parameters:

Name Type Description Default
timeout

Maximum time to wait (in seconds) for all tasks to complete. None means wait indefinitely.

None

Returns:

Name Type Description
bool

True if all tasks completed, False if timeout occurred

Source code in toolboxv2/utils/toolbox.py
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
def wait_for_bg_tasks(self, timeout=None):
    """
    Wait for all background tasks to complete.

    Args:
        timeout: Maximum time to wait (in seconds) for all tasks to complete.
                 None means wait indefinitely.

    Returns:
        bool: True if all tasks completed, False if timeout occurred
    """
    active_tasks = [t for t in self.bg_tasks if t.is_alive()]

    for task in active_tasks:
        task.join(timeout=timeout)
        if task.is_alive():
            return False

    return True
ws_broadcast(channel_id, payload, source_conn_id='python_broadcast') async

Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

Parameters:

Name Type Description Default
channel_id str

Der Kanal, an den gesendet werden soll.

required
payload dict

Ein Dictionary, das als JSON gesendet wird.

required
source_conn_id optional

Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.

'python_broadcast'
Source code in toolboxv2/utils/toolbox.py
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
    """
    Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

    Args:
        channel_id: Der Kanal, an den gesendet werden soll.
        payload: Ein Dictionary, das als JSON gesendet wird.
        source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Broadcast-Methode auf
        await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
    except Exception as e:
        self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)
ws_send(conn_id, payload) async

Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

Parameters:

Name Type Description Default
conn_id str

Die eindeutige ID der Zielverbindung.

required
payload dict

Ein Dictionary, das als JSON gesendet wird.

required
Source code in toolboxv2/utils/toolbox.py
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
async def ws_send(self, conn_id: str, payload: dict):
    """
    Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

    Args:
        conn_id: Die eindeutige ID der Zielverbindung.
        payload: Ein Dictionary, das als JSON gesendet wird.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
        await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
    except Exception as e:
        self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

Code

Source code in toolboxv2/utils/security/cryp.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()
decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name Type Description Default
encrypted_text_hex str

Der verschlüsselte Text als Hex-String.

required
private_key_str str

Der private Schlüssel als String.

required

Returns:

Name Type Description
str str

Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"
decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
encrypted_text str

Der zu entschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required
to_str bool

default true returns str if false returns bytes

True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()
encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
public_key_str str

Der öffentliche Schlüssel als String oder im pem format.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"
encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"
generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name Type Description Default
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key
generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name Type Description Default
length int

Die Länge der zu generierenden Zeichenkette.

required

Returns:

Name Type Description
str str

Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py
81
82
83
84
85
86
87
88
89
90
91
92
@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)
generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name Type Description
int int

Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py
114
115
116
117
118
119
120
121
122
@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)
generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name Type Description
str str or bytes

Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
140
141
142
143
144
145
146
147
148
149
150
151
@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key
load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name Type Description Default
directory str

Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

'keys'

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type Description
FileNotFoundError

Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key
one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name Type Description Default
text str

Der zu hashende Text.

required
salt str

Der Salt-Wert.

''
pepper str

Der Pepper-Wert.

''
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Name Type Description
str str

Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()
pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name Type Description Default
pem_key str

Der PEM-kodierte öffentliche Schlüssel.

required

Returns:

Name Type Description
PublicKey

Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py
435
436
437
438
439
440
441
442
443
444
445
446
447
@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key
public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name Type Description Default
public_key PublicKey

Das PublicKey-Objekt.

required

Returns:

Name Type Description
str

Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()
save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name Type Description Default
public_key str

Der öffentliche Schlüssel im PEM-Format

required
private_key str

Der private Schlüssel im PEM-Format

required
directory str

Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen

'keys'
Source code in toolboxv2/utils/security/cryp.py
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)

MainTool

Source code in toolboxv2/utils/system/main_tool.py
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()
__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")
__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py
174
175
176
177
178
179
180
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self
get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py
167
168
169
def get_version(self) -> str:
    """"Returns the version"""
    return self.version
webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py
164
165
def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""

Result

Source code in toolboxv2/utils/system/types.py
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task
__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py
643
644
645
646
647
648
649
650
651
652
653
def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult
binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)
cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py
738
739
740
741
742
743
744
745
746
747
748
def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result
file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name Type Description Default
data

File data as bytes or base64 string

required
filename

Name of the file for download

required
content_type

MIME type of the file (auto-detected if None)

None
info

Response info text

'OK'
interface

Target interface

remote

Returns:

Type Description

Result object configured for file download

Source code in toolboxv2/utils/system/types.py
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)
get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py
750
751
752
def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type
is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py
754
755
756
def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None
json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py
970
971
972
973
974
975
976
977
978
979
980
981
982
983
@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)
redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)
sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name Type Description Default
stream_generator Any

A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.

required
info str

Optional help text for the Result.

'OK'
interface ToolBoxInterfaces

Optional ToolBoxInterface to target.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional cleanup function to run when the stream ends or is cancelled.

None
#http_headers

Optional dictionary of custom HTTP headers for the SSE response.

required

Returns:

Type Description

A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )
stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name Type Description Default
stream_generator Any

Any stream source (async generator, sync generator, iterable, or single item).

required
content_type str

Content-Type header (default: text/event-stream for SSE).

'text/event-stream'
headers dict | None

Additional HTTP headers for the response.

None
info str

Help text for the result.

'OK'
interface ToolBoxInterfaces

Interface to send data to.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional function for cleanup.

None

Returns:

Type Description

A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)
text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)
typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py
667
668
669
670
671
672
673
674
675
676
async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data
typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py
655
656
657
658
659
660
661
662
663
664
665
def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data
typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance
typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py
705
706
707
708
709
710
711
712
713
714
715
716
@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

Singleton

Singleton metaclass for ensuring only one instance of a class.

Source code in toolboxv2/utils/singelton_class.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
class Singleton(type):
    """
    Singleton metaclass for ensuring only one instance of a class.
    """

    _instances = {}
    _kwargs = {}
    _args = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super().__call__(*args, **kwargs)
            cls._args[cls] = args
            cls._kwargs[cls] = kwargs
        return cls._instances[cls]

Spinner

Enhanced Spinner with tqdm-like line rendering.

Source code in toolboxv2/utils/extras/Style.py
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
class Spinner:
    """
    Enhanced Spinner with tqdm-like line rendering.
    """
    SYMBOL_SETS = {
        "c": ["◐", "◓", "◑", "◒"],
        "b": ["▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"],
        "d": ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"],
        "w": ["🌍", "🌎", "🌏"],
        "s": ["🌀   ", " 🌀  ", "  🌀 ", "   🌀", "  🌀 ", " 🌀  "],
        "+": ["+", "x"],
        "t": ["✶", "✸", "✹", "✺", "✹", "✷"]
    }

    def __init__(
        self,
        message: str = "Loading...",
        delay: float = 0.1,
        symbols=None,
        count_down: bool = False,
        time_in_s: float = 0
    ):
        """Initialize spinner with flexible configuration."""
        # Resolve symbol set.
        if isinstance(symbols, str):
            symbols = self.SYMBOL_SETS.get(symbols, None)

        # Default symbols if not provided.
        if symbols is None:
            symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

        # Test mode symbol set.
        if 'unittest' in sys.argv[0]:
            symbols = ['#', '=', '-']

        self.spinner = itertools.cycle(symbols)
        self.delay = delay
        self.message = message
        self.running = False
        self.spinner_thread = None
        self.max_t = time_in_s
        self.contd = count_down

        # Rendering management.
        self._is_primary = False
        self._start_time = 0

        # Central manager.
        self.manager = SpinnerManager()

    def _generate_render_line(self):
        """Generate the primary render line."""
        current_time = time.time()
        if self.contd:
            remaining = max(0, self.max_t - (current_time - self._start_time))
            time_display = f"{remaining:.2f}"
        else:
            time_display = f"{current_time - self._start_time:.2f}"

        symbol = next(self.spinner)
        return f"{symbol} {self.message} | {time_display}"

    def _generate_secondary_info(self):
        """Generate secondary spinner info for additional spinners."""
        return f"{self.message}"

    def __enter__(self):
        """Start the spinner."""
        self.running = True
        self._start_time = time.time()
        self.manager.register_spinner(self)
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        """Stop the spinner."""
        self.running = False
        self.manager.unregister_spinner(self)
        # Clear the spinner's line if it was the primary spinner.
        if self._is_primary:
            sys.stdout.write("\r\033[K")
            sys.stdout.flush()
__enter__()

Start the spinner.

Source code in toolboxv2/utils/extras/Style.py
644
645
646
647
648
649
def __enter__(self):
    """Start the spinner."""
    self.running = True
    self._start_time = time.time()
    self.manager.register_spinner(self)
    return self
__exit__(exc_type, exc_value, exc_traceback)

Stop the spinner.

Source code in toolboxv2/utils/extras/Style.py
651
652
653
654
655
656
657
658
def __exit__(self, exc_type, exc_value, exc_traceback):
    """Stop the spinner."""
    self.running = False
    self.manager.unregister_spinner(self)
    # Clear the spinner's line if it was the primary spinner.
    if self._is_primary:
        sys.stdout.write("\r\033[K")
        sys.stdout.flush()
__init__(message='Loading...', delay=0.1, symbols=None, count_down=False, time_in_s=0)

Initialize spinner with flexible configuration.

Source code in toolboxv2/utils/extras/Style.py
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
def __init__(
    self,
    message: str = "Loading...",
    delay: float = 0.1,
    symbols=None,
    count_down: bool = False,
    time_in_s: float = 0
):
    """Initialize spinner with flexible configuration."""
    # Resolve symbol set.
    if isinstance(symbols, str):
        symbols = self.SYMBOL_SETS.get(symbols, None)

    # Default symbols if not provided.
    if symbols is None:
        symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

    # Test mode symbol set.
    if 'unittest' in sys.argv[0]:
        symbols = ['#', '=', '-']

    self.spinner = itertools.cycle(symbols)
    self.delay = delay
    self.message = message
    self.running = False
    self.spinner_thread = None
    self.max_t = time_in_s
    self.contd = count_down

    # Rendering management.
    self._is_primary = False
    self._start_time = 0

    # Central manager.
    self.manager = SpinnerManager()

TBEF

Automatic generated by ToolBox v = 0.1.22

clis

api
cleanup_build_files()

Clean build artifacts

Source code in toolboxv2/utils/clis/api.py
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
def cleanup_build_files():
    """Clean build artifacts"""
    print_box_header("Cleaning Build Artifacts", "🧹")
    print_box_footer()

    from toolboxv2 import tb_root_dir
    src_core_path = tb_root_dir / "src-core"
    target_path = src_core_path / "target"

    if target_path.exists():
        try:
            with Spinner("Running cargo clean", symbols="+") as s:
                try:
                    subprocess.run(
                        ["cargo", "clean"],
                        cwd=src_core_path,
                        check=True,
                        capture_output=True
                    )
                except subprocess.CalledProcessError:
                    s.message = "Manually removing build directories"
                    for item in target_path.iterdir():
                        if item.is_dir() and item.name != ".rustc_info.json":
                            shutil.rmtree(item)

            print()
            print_status("Build artifacts cleaned successfully", "success")
            return True

        except Exception as e:
            print()
            print_status(f"Failed to clean build files: {e}", "error")
            return False
    else:
        print_status(f"Build directory not found: {target_path}", "warning")
        return True
cli_api_runner()

Main CLI entry point

Source code in toolboxv2/utils/clis/api.py
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
def cli_api_runner():
    """Main CLI entry point"""
    parser = argparse.ArgumentParser(
        prog='tb api',
        description='🚀 Platform-Agnostic Rust API Server Manager',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
╔════════════════════════════════════════════════════════════════════════════╗
║                           Command Examples                                 ║
╠════════════════════════════════════════════════════════════════════════════╣
║                                                                            ║
║  Build & Development:                                                      ║
║    $ tb api build                    # Build release executable            ║
║    $ tb api debug                    # Run with hot reload                 ║
║    $ tb api clean                    # Clean build artifacts               ║
║    $ tb api remove-exe               # Remove compiled executable          ║
║    $ tb api visual-test              # Run visual UI component test        ║
║                                                                            ║
║  Server Management:                                                        ║
║    $ tb api start                    # Start server                        ║
║    $ tb api stop                     # Stop server                         ║
║    $ tb api status                   # Check server status                 ║
║                                                                            ║
║  Advanced Options:                                                         ║
║    $ tb api start --posix-zdt        # Start with zero-downtime (Linux)    ║
║    $ tb api start --exe /path --version 1.0.0                              ║
║                                                                            ║
║  Zero-Downtime Updates (Linux/macOS):                                      ║
║    $ tb api update --exe /new/path --version 1.1.0 --posix-zdt             ║
║                                                                            ║
║  Graceful Updates (All Platforms):                                         ║
║    $ tb api update --exe /new/path --version 1.1.0                         ║
║                                                                            ║
╚════════════════════════════════════════════════════════════════════════════╝

Server Configuration:
  Host: {SERVER_HOST}
  Port: {SERVER_PORT}

Zero-Downtime Updates:
  Available on Linux and macOS using --posix-zdt flag
  Uses socket file descriptor passing for seamless updates
  Windows uses graceful restart (brief downtime)
        """
    )

    subparsers = parser.add_subparsers(dest="action", required=False, help="Available actions")

    # Build commands
    subparsers.add_parser('build', help='Build the Rust project in release mode')
    subparsers.add_parser('debug', help='Run in debug mode with hot reload')
    subparsers.add_parser('clean', help='Clean build artifacts')
    subparsers.add_parser('remove-exe', help='Remove release executable')
    subparsers.add_parser('visual-test', help='Run visual test for UI components')

    # Server management commands
    actions = {
        'start': 'Start the API server',
        'stop': 'Stop the running server',
        'update': 'Update server to new version',
        'status': 'Display server status'
    }

    for action, help_text in actions.items():
        p = subparsers.add_parser(action, help=help_text)

        if action in ['start', 'update', 'status']:
            p.add_argument(
                '--posix-zdt',
                action='store_true',
                help='(Linux/macOS) Enable POSIX zero-downtime updates via socket passing'
            )

        if action in ['start', 'update']:
            p.add_argument(
                '--exe',
                type=str,
                help='Path to server executable'
            )
            p.add_argument(
                '--version',
                type=str,
                default='unknown',
                help='Version string for the server'
            )

    args = parser.parse_args()

    # Handle simple actions
    if args.action == 'build':
        handle_build()
        return

    if args.action == 'clean':
        cleanup_build_files()
        return

    if args.action == 'remove-exe':
        remove_release_executable()
        return

    if args.action == 'debug':
        handle_debug()
        return

    if args.action == 'visual-test':
        run_visual_test()
        return

    if not args.action:
        # Default to status if no action provided
        show_server_status(False)
        return

    # Handle server management
    manage_server(
        action=args.action,
        executable_path=getattr(args, 'exe', None),
        version_str=getattr(args, 'version', 'unknown'),
        use_posix_zdt=getattr(args, 'posix_zdt', False)
    )
ensure_socket_and_fd_file_posix(host, port, backlog, fd_file_path)

Create socket and FD file for POSIX zero-downtime updates

Source code in toolboxv2/utils/clis/api.py
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
def ensure_socket_and_fd_file_posix(host, port, backlog, fd_file_path):
    """Create socket and FD file for POSIX zero-downtime updates"""
    if os.path.exists(fd_file_path):
        print_status(f"Stale FD file found: {fd_file_path}", "warning")
        print_status("Removing to create new socket", "info")
        with contextlib.suppress(OSError):
            os.remove(fd_file_path)

    try:
        print_status("Creating listening socket", "progress")

        server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

        fd_num = server_socket.fileno()

        # Make socket inheritable
        if hasattr(os, 'set_inheritable'):
            os.set_inheritable(fd_num, True)
        else:
            import fcntl
            flags = fcntl.fcntl(fd_num, fcntl.F_GETFD)
            fcntl.fcntl(fd_num, fcntl.F_SETFD, flags & ~fcntl.FD_CLOEXEC)

        server_socket.bind((host, port))
        server_socket.listen(backlog)

        # Save FD to file
        with open(fd_file_path, 'w') as f:
            f.write(str(fd_num))
        os.chmod(fd_file_path, 0o600)

        print_status(f"Socket created - FD {fd_num} saved to {fd_file_path}", "success")
        return server_socket, fd_num

    except Exception as e:
        print_status(f"Failed to create listening socket: {e}", "error")
        if 'server_socket' in locals():
            server_socket.close()
        return None, None
get_executable_name_with_extension(base_name=DEFAULT_EXECUTABLE_NAME)

Get platform-specific executable name

Source code in toolboxv2/utils/clis/api.py
59
60
61
62
63
def get_executable_name_with_extension(base_name=DEFAULT_EXECUTABLE_NAME):
    """Get platform-specific executable name"""
    if platform.system().lower() == "windows":
        return f"{base_name}.exe"
    return base_name
get_executable_path()

Find the release executable in standard locations

Source code in toolboxv2/utils/clis/api.py
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
def get_executable_path():
    """Find the release executable in standard locations"""
    exe_name = get_executable_name_with_extension()
    from toolboxv2 import tb_root_dir

    search_paths = [
        tb_root_dir / Path("bin") / exe_name,
        tb_root_dir / Path("src-core") / exe_name,
        tb_root_dir / exe_name,
        tb_root_dir / Path("src-core") / "target" / "release" / exe_name,
    ]

    for path in search_paths:
        if path.exists() and path.is_file():
            return path.resolve()

    return None
handle_build()

Build the Rust project

Source code in toolboxv2/utils/clis/api.py
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
def handle_build():
    """Build the Rust project"""
    print_box_header("Building Rust API Server", "🔨")
    print_box_content("Compiler: Cargo (Rust)", "info")
    print_box_content("Mode: Release", "info")
    print_box_footer()

    from toolboxv2 import tb_root_dir

    try:
        with Spinner("Compiling Rust project", symbols="t") as s:
            result = subprocess.run(
                ["cargo", "build", "--release"],
                cwd=tb_root_dir / "src-core",
                check=True,
                capture_output=True,
                text=True
            )

        print()
        print_status("Build completed successfully", "success")

        # Copy executable
        exe_path = get_executable_path()
        if exe_path:
            bin_dir = tb_root_dir / "bin"
            bin_dir.mkdir(exist_ok=True)

            try:
                dest_path = bin_dir / exe_path.name
                shutil.copy(exe_path, dest_path)
                print_status(f"Executable copied to: {dest_path}", "success")
            except Exception as e:
                print_status(f"Warning: Failed to copy to bin: {e}", "warning")

                # Fallback to ubin
                ubin_dir = tb_root_dir / "ubin"
                ubin_dir.mkdir(exist_ok=True)
                dest_path = ubin_dir / exe_path.name

                try:
                    shutil.copy(exe_path, dest_path)
                    print_status(f"Copied to fallback location: {dest_path}", "info")
                except Exception as e_ubin:
                    print_status(f"Error copying to ubin: {e_ubin}", "error")

    except subprocess.CalledProcessError as e:
        print()
        print_box_header("Build Failed", "✗")
        print_box_content("Compilation errors occurred", "error")
        print_box_footer()
        print("\nError output:")
        print(e.stderr)

    except FileNotFoundError:
        print()
        print_box_header("Build Failed", "✗")
        print_box_content("'cargo' command not found", "error")
        print_box_content("Is Rust installed and in your PATH?", "info")
        print_box_footer()
handle_debug()

Run server in debug mode with hot reload

Source code in toolboxv2/utils/clis/api.py
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
def handle_debug():
    """Run server in debug mode with hot reload"""
    print_box_header("Debug Mode with Hot Reload", "🔥")
    print_box_content("Watching for file changes", "info")
    print_box_content("Press Ctrl+C to stop", "info")
    print_box_footer()

    from toolboxv2 import tb_root_dir
    src_core_path = tb_root_dir / "src-core"

    # Check if cargo-watch is installed
    try:
        subprocess.run(
            ["cargo", "watch", "--version"],
            check=True,
            cwd=src_core_path,
            capture_output=True
        )
    except Exception:
        print_status("cargo-watch not installed", "warning")
        print_status("Installing cargo-watch...", "progress")

        try:
            with Spinner("Installing cargo-watch", symbols="t"):
                subprocess.run(
                    ["cargo", "install", "cargo-watch"],
                    check=True,
                    cwd=src_core_path,
                    capture_output=True
                )
            print()
            print_status("cargo-watch installed successfully", "success")
        except subprocess.CalledProcessError as e:
            print()
            print_status("Failed to install cargo-watch", "error")
            print_status("Falling back to standard debug mode", "warning")
            print()

            # Fallback to standard debug
            print_separator("═")
            print("  Running in standard debug mode")
            print_separator("═")
            print()

            try:
                subprocess.run(["cargo", "run"], cwd=src_core_path)
            except KeyboardInterrupt:
                print()
                print_status("Debug mode stopped", "info")
            return

    # Run with hot reload
    print_separator("═")
    print("  Starting Hot Reload Session")
    print_separator("═")
    print()

    try:
        subprocess.run(["cargo", "watch", "-x", "run"], cwd=src_core_path)
    except KeyboardInterrupt:
        print()
        print_status("Hot reload stopped", "info")
    except subprocess.CalledProcessError as e:
        print_status(f"Hot reload failed: {e}", "error")
is_process_running(pid)

Check if process is running

Source code in toolboxv2/utils/clis/api.py
113
114
115
116
117
118
119
120
def is_process_running(pid):
    """Check if process is running"""
    if pid is None or psutil is None:
        return False
    try:
        return psutil.pid_exists(int(pid))
    except (ValueError, TypeError):
        return False
manage_server(action, executable_path=None, version_str='unknown', use_posix_zdt=False)

Main server management dispatcher

Source code in toolboxv2/utils/clis/api.py
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
def manage_server(action: str, executable_path: str = None, version_str: str = "unknown", use_posix_zdt: bool = False):
    """Main server management dispatcher"""

    if action == "start":
        if not executable_path:
            executable_path = get_executable_path()

        if not executable_path:
            print_box_header("Executable Not Found", "✗")
            print_box_content("No compiled executable found", "error")
            print_box_content("Build first with: tb api build", "info")
            print_box_footer()
            return False

        start_new_server(executable_path, version_str, use_posix_zdt)

    elif action == "stop":
        pid, _, _ = read_server_state()
        if stop_process(pid):
            write_server_state(None, None, None)

            # Clean up FD file if exists
            is_posix = platform.system().lower() != "windows"
            if is_posix and os.path.exists(PERSISTENT_FD_FILE):
                print()
                print_status(f"Note: Stale FD file exists: {PERSISTENT_FD_FILE}", "warning")
                print_status("Consider removing it before next start", "info")

    elif action == "update":
        if not executable_path:
            print_box_header("Update Failed", "✗_")
            print_box_content("New executable path required (--exe)", "error")
            print_box_footer()
            return False

        if not version_str or version_str == "unknown":
            print_box_header("Update Failed", "✗_")
            print_box_content("Version string required (--version)", "error")
            print_box_footer()
            return False

        update_server(executable_path, version_str, use_posix_zdt)

    elif action == "status":
        show_server_status(use_posix_zdt)

    return True
read_server_state(state_file=SERVER_STATE_FILE)

Read server state from file

Source code in toolboxv2/utils/clis/api.py
85
86
87
88
89
90
91
92
93
94
def read_server_state(state_file=SERVER_STATE_FILE):
    """Read server state from file"""
    try:
        if os.path.exists(state_file):
            with open(state_file) as f:
                state = json.load(f)
                return state.get('pid'), state.get('version'), state.get('executable_path')
        return None, None, None
    except Exception:
        return None, None, None
remove_release_executable()

Remove release executable

Source code in toolboxv2/utils/clis/api.py
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
def remove_release_executable():
    """Remove release executable"""
    print_box_header("Removing Release Executable", "🗑️")
    print_box_footer()

    from toolboxv2 import tb_root_dir
    src_core_path = tb_root_dir / "src-core"
    exe_name = get_executable_name_with_extension()

    paths_to_remove = [
        src_core_path / exe_name,
        src_core_path / "target" / "release" / exe_name
    ]

    removed_count = 0

    for path in paths_to_remove:
        if path.exists():
            try:
                path.unlink()
                print_status(f"Removed: {path}", "success")
                removed_count += 1
            except Exception as e:
                print_status(f"Failed to remove {path}: {e}", "error")

    if removed_count == 0:
        print_status("No executables found to remove", "warning")
    else:
        print()
        print_status(f"Removed {removed_count} executable(s)", "success")

    return True
show_server_status(use_posix_zdt)

Display current server status

Source code in toolboxv2/utils/clis/api.py
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
def show_server_status(use_posix_zdt: bool):
    """Display current server status"""
    pid, ver, exe = read_server_state()

    print_box_header("Server Status", "🖥️")
    print()

    if is_process_running(pid):
        # Running - show full details
        columns = [
            ("Property", 15),
            ("Value", 50)
        ]
        widths = [w for _, w in columns]

        print_table_header(columns, widths)
        print_table_row(["Status", "RUNNING"], widths, ["white", "green"])
        print_table_row(["PID", str(pid)], widths, ["white", "grey"])
        print_table_row(["Version", ver or 'N/A'], widths, ["white", "yellow"])
        print_table_row(["Executable", str(exe) if exe else 'N/A'], widths, ["white", "cyan"])
        print_table_row(["Host", SERVER_HOST], widths, ["white", "blue"])
        print_table_row(["Port", str(SERVER_PORT)], widths, ["white", "blue"])

        # Check for POSIX ZDT
        is_posix = platform.system().lower() != "windows"
        if is_posix and os.path.exists(PERSISTENT_FD_FILE) and use_posix_zdt:
            try:
                with open(PERSISTENT_FD_FILE) as f:
                    fd_val = f.read().strip()
                print_table_row(["ZDT Mode", f"Active (FD: {fd_val})"], widths, ["white", "green"])
            except Exception:
                pass

        print()
        print_status("Server is healthy and running", "success")

    else:
        print_box_content("Status: STOPPED", "error")
        if pid:
            print_box_content(f"Stale PID in state: {pid}", "warning")
        print()
        print_status("Server is not running", "warning")

    print_box_footer()
start_new_server(executable_path, version_str, use_posix_zdt)

Start a new server instance

Source code in toolboxv2/utils/clis/api.py
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
def start_new_server(executable_path, version_str, use_posix_zdt):
    """Start a new server instance"""
    current_pid, _, _ = read_server_state()

    if is_process_running(current_pid):
        print_box_header("Server Already Running", "⚠")
        print_box_content(f"PID: {current_pid}", "warning")
        print_box_content("Use 'stop' or 'update' command", "info")
        print_box_footer()
        return

    print_box_header(f"Starting Server v{version_str}", "🚀")
    print_box_content(f"Executable: {executable_path}", "info")
    print_box_content(f"Host: {SERVER_HOST}:{SERVER_PORT}", "network")

    is_posix = platform.system().lower() != "windows"

    if is_posix and use_posix_zdt:
        print_box_content("Mode: POSIX Zero-Downtime", "info")
    else:
        print_box_content("Mode: Standard Start", "info")

    print_box_footer()

    process = None
    socket_obj = None

    with Spinner(f"Launching server", symbols="d") as s:
        if is_posix and use_posix_zdt:
            socket_obj, fd = ensure_socket_and_fd_file_posix(
                SERVER_HOST, SERVER_PORT, SOCKET_BACKLOG, PERSISTENT_FD_FILE
            )
            if fd is not None:
                process = start_rust_server_posix(executable_path, fd)
        else:
            process = start_rust_server_windows(executable_path)

        time.sleep(2)  # Stabilization period

    # Close parent's socket handle
    if socket_obj:
        socket_obj.close()

    print()

    if process and process.poll() is None:
        write_server_state(process.pid, version_str, executable_path)

        print_box_header("Server Started", "✓")
        print_box_content(f"Version: {version_str}", "success")
        print_box_content(f"PID: {process.pid}", "success")
        print_box_content(f"Port: {SERVER_PORT}", "success")
        print_box_footer()
    else:
        print_box_header("Server Failed to Start", "✗")
        print_box_content("Check logs for details", "error")
        print_box_footer()
        write_server_state(None, None, None)
start_rust_server_posix(executable_path, persistent_fd)

Start Rust server on POSIX with socket FD passing

Source code in toolboxv2/utils/clis/api.py
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
def start_rust_server_posix(executable_path: str, persistent_fd: int):
    """Start Rust server on POSIX with socket FD passing"""
    abs_path = Path(executable_path).resolve()

    env = os.environ.copy()
    env["PERSISTENT_LISTENER_FD"] = str(persistent_fd)

    print_status(f"Starting {abs_path.name} with FD {persistent_fd}", "server")

    try:
        return subprocess.Popen(
            [str(abs_path)],
            cwd=abs_path.parent,
            env=env,
            pass_fds=[persistent_fd]
        )
    except Exception as e:
        print_status(f"Failed to start server: {e}", "error")
        return None
start_rust_server_windows(executable_path)

Start Rust server on Windows

Source code in toolboxv2/utils/clis/api.py
219
220
221
222
223
224
225
226
227
228
229
230
def start_rust_server_windows(executable_path: str):
    """Start Rust server on Windows"""
    abs_path = Path(executable_path).resolve()

    print_status(f"Starting {abs_path.name}", "server")
    print_status(f"Working directory: {abs_path.parent}", "info")

    try:
        return subprocess.Popen([str(abs_path)], cwd=abs_path.parent)
    except Exception as e:
        print_status(f"Failed to start server: {e}", "error")
        return None
stop_process(pid, timeout=10)

Stop process gracefully with timeout

Source code in toolboxv2/utils/clis/api.py
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
def stop_process(pid, timeout=10):
    """Stop process gracefully with timeout"""
    if not is_process_running(pid):
        print_status(f"Process {pid} not running", "warning")
        return True

    print_box_header(f"Stopping Process", "⏹️")
    print_box_content(f"PID: {pid}", "info")
    print_box_content(f"Timeout: {timeout}s", "info")
    print_box_footer()

    with Spinner(f"Stopping process {pid}", symbols="+", time_in_s=timeout, count_down=True) as s:
        try:
            proc = psutil.Process(int(pid))
            proc.terminate()
            proc.wait(timeout)
        except psutil.TimeoutExpired:
            s.message = f"Force killing process {pid}"
            proc.kill()
        except psutil.NoSuchProcess:
            pass
        except Exception as e:
            print()
            print_status(f"Error stopping process {pid}: {e}", "error")
            return False

    print()
    print_status(f"Process {pid} stopped successfully", "success")
    return True
update_server(new_executable_path, new_version, use_posix_zdt)

High-level update function

Source code in toolboxv2/utils/clis/api.py
391
392
393
394
395
396
397
398
399
400
401
402
def update_server(new_executable_path: str, new_version: str, use_posix_zdt: bool):
    """High-level update function"""
    is_posix = platform.system().lower() != "windows"

    if is_posix and use_posix_zdt:
        return update_server_posix(new_executable_path, new_version)
    else:
        if use_posix_zdt and not is_posix:
            print_status("--posix-zdt flag ignored on Windows", "warning")
            print_status("Using graceful restart instead", "info")
            print()
        return update_server_graceful_restart(new_executable_path, new_version)
update_server_graceful_restart(new_executable_path, new_version)

Perform graceful restart update

Source code in toolboxv2/utils/clis/api.py
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
def update_server_graceful_restart(new_executable_path: str, new_version: str):
    """Perform graceful restart update"""
    print_box_header(f"Graceful Restart to v{new_version}", "🔄")
    print_box_content("Method: Stop & Start", "info")
    print_box_footer()

    old_pid, old_version, _ = read_server_state()

    # Step 1: Stop old server
    print_separator("═")
    print("  PHASE 1: Stopping Current Server")
    print_separator("═")
    print()

    if not stop_process(old_pid):
        print_status("Failed to stop old server", "error")
        print_status("Update aborted to prevent conflicts", "error")
        return False

    # Step 2: Start new server
    print()
    print_separator("═")
    print("  PHASE 2: Starting New Server")
    print_separator("═")
    print()

    start_new_server(new_executable_path, new_version, False)

    return True
update_server_posix(new_executable_path, new_version)

Perform POSIX zero-downtime update

Source code in toolboxv2/utils/clis/api.py
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
def update_server_posix(new_executable_path: str, new_version: str):
    """Perform POSIX zero-downtime update"""
    print_box_header(f"Zero-Downtime Update to v{new_version}", "🔄")
    print_box_content("Method: POSIX Socket Passing", "info")
    print_box_footer()

    old_pid, old_version, _ = read_server_state()

    if not os.path.exists(PERSISTENT_FD_FILE):
        print_status(f"FD file '{PERSISTENT_FD_FILE}' not found", "error")
        print_status("Cannot perform zero-downtime update", "error")
        return False

    try:
        with open(PERSISTENT_FD_FILE) as f:
            persistent_fd = int(f.read().strip())
        print_status(f"Using FD {persistent_fd} for socket passing", "info")
    except Exception as e:
        print_status(f"Error reading FD from file: {e}", "error")
        return False

    # Step 1: Start new server
    print()
    print_separator("═")
    print("  PHASE 1: Starting New Server")
    print_separator("═")
    print()

    with Spinner(f"Starting new server v{new_version}", symbols="d") as s:
        new_process = start_rust_server_posix(new_executable_path, persistent_fd)
        time.sleep(3)

    print()

    if new_process is None or new_process.poll() is not None:
        print_status("New server process died on startup", "error")
        print_status("Update aborted", "error")
        return False

    print_status(f"New server started (PID: {new_process.pid})", "success")

    # Step 2: Stop old server
    print()
    print_separator("═")
    print("  PHASE 2: Stopping Old Server")
    print_separator("═")
    print()

    if stop_process(old_pid):
        write_server_state(new_process.pid, new_version, new_executable_path)

        print()
        print_box_header("Update Complete", "✓")
        print_box_content(f"Old Version: {old_version} (PID: {old_pid})", "info")
        print_box_content(f"New Version: {new_version} (PID: {new_process.pid})", "success")
        print_box_content("Zero downtime achieved!", "success")
        print_box_footer()
        return True
    else:
        print_status("Failed to stop old process", "error")
        print_status("Manual intervention may be required", "warning")
        stop_process(new_process.pid)
        return False
write_server_state(pid, server_version, executable_path, state_file=SERVER_STATE_FILE)

Write server state to file

Source code in toolboxv2/utils/clis/api.py
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def write_server_state(pid, server_version, executable_path, state_file=SERVER_STATE_FILE):
    """Write server state to file"""
    if executable_path is None:
        executable_path = ''
    try:
        state = {
            'pid': pid,
            'version': server_version,
            'executable_path': str(Path(executable_path).resolve())
        }
        with open(state_file, 'w') as f:
            json.dump(state, f, indent=4)
    except Exception as e:
        print_status(f"Error writing server state: {e}", "error")
cli_printing
Colors

ANSI color codes for terminal styling

Source code in toolboxv2/utils/clis/cli_printing.py
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
class Colors:
    """ANSI color codes for terminal styling"""
    # Basic colors
    BLACK = '\033[30m'
    RED = '\033[31m'
    GREEN = '\033[32m'
    YELLOW = '\033[33m'
    BLUE = '\033[34m'
    MAGENTA = '\033[35m'
    CYAN = '\033[36m'
    WHITE = '\033[37m'
    GREY = '\033[90m'

    # Bright colors
    BRIGHT_RED = '\033[91m'
    BRIGHT_GREEN = '\033[92m'
    BRIGHT_YELLOW = '\033[93m'
    BRIGHT_BLUE = '\033[94m'
    BRIGHT_MAGENTA = '\033[95m'
    BRIGHT_CYAN = '\033[96m'
    BRIGHT_WHITE = '\033[97m'

    # Styles
    BOLD = '\033[1m'
    DIM = '\033[2m'
    ITALIC = '\033[3m'
    UNDERLINE = '\033[4m'
    BLINK = '\033[5m'
    REVERSE = '\033[7m'

    # Background colors
    BG_BLACK = '\033[40m'
    BG_RED = '\033[41m'
    BG_GREEN = '\033[42m'
    BG_YELLOW = '\033[43m'
    BG_BLUE = '\033[44m'
    BG_MAGENTA = '\033[45m'
    BG_CYAN = '\033[46m'
    BG_WHITE = '\033[47m'

    # Reset
    RESET = '\033[0m'
main()

Entry point for running visual test directly

Source code in toolboxv2/utils/clis/cli_printing.py
437
438
439
def main():
    """Entry point for running visual test directly"""
    run_visual_test()
print_box_content(text, style='', width=76, auto_wrap=True)

Print content with minimal styled prefix

Source code in toolboxv2/utils/clis/cli_printing.py
143
144
145
146
147
148
149
150
151
152
153
154
155
156
def print_box_content(text: str, style: str = "", width: int = 76, auto_wrap: bool = True):
    """Print content with minimal styled prefix"""
    style_config = {
        'success': {'icon': '✓', 'color': Colors.GREEN},
        'error': {'icon': '✗', 'color': Colors.RED},
        'warning': {'icon': '⚠', 'color': Colors.YELLOW},
        'info': {'icon': 'ℹ', 'color': Colors.BLUE},
    }

    if style in style_config:
        config = style_config[style]
        print(f"  {config['color']}{config['icon']}{Colors.RESET} {text}")
    else:
        print(f"  {text}")

Print a minimal footer

Source code in toolboxv2/utils/clis/cli_printing.py
136
137
138
def print_box_footer(width: int = 76):
    """Print a minimal footer"""
    print()
print_box_header(title, icon='ℹ', width=76)

Print a minimal styled header

Source code in toolboxv2/utils/clis/cli_printing.py
129
130
131
132
133
def print_box_header(title: str, icon: str = "ℹ", width: int = 76):
    """Print a minimal styled header"""
    print()
    print(f"{Colors.BOLD}{icon} {title}{Colors.RESET}")
    print(f"{Colors.DIM}{'─' * width}{Colors.RESET}")
print_code_block(code, language='text', width=76, show_line_numbers=False)

Print code block with minimal syntax highlighting

Source code in toolboxv2/utils/clis/cli_printing.py
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
def print_code_block(code: str, language: str = "text", width: int = 76, show_line_numbers: bool = False):
    """Print code block with minimal syntax highlighting"""
    import json

    if language.lower() in ['json']:
        try:
            parsed = json.loads(code) if isinstance(code, str) else code
            formatted = json.dumps(parsed, indent=2)
            lines = formatted.split('\n')
        except:
            lines = code.split('\n')
    elif language.lower() in ['yaml', 'yml']:
        lines = code.split('\n')
        formatted_lines = []
        for line in lines:
            if ':' in line and not line.strip().startswith('#'):
                key, value = line.split(':', 1)
                formatted_lines.append(f"{Colors.CYAN}{key}{Colors.RESET}:{value}")
            elif line.strip().startswith('#'):
                formatted_lines.append(f"{Colors.DIM}{line}{Colors.RESET}")
            else:
                formatted_lines.append(line)
        lines = formatted_lines
    elif language.lower() in ['toml']:
        lines = code.split('\n')
        formatted_lines = []
        for line in lines:
            if line.strip().startswith('[') and line.strip().endswith(']'):
                formatted_lines.append(f"{Colors.BOLD}{line}{Colors.RESET}")
            elif '=' in line and not line.strip().startswith('#'):
                key, value = line.split('=', 1)
                formatted_lines.append(f"{Colors.CYAN}{key}{Colors.RESET}={value}")
            elif line.strip().startswith('#'):
                formatted_lines.append(f"{Colors.DIM}{line}{Colors.RESET}")
            else:
                formatted_lines.append(line)
        lines = formatted_lines
    elif language.lower() in ['env', 'dotenv']:
        lines = code.split('\n')
        formatted_lines = []
        for line in lines:
            if '=' in line and not line.strip().startswith('#'):
                key, value = line.split('=', 1)
                formatted_lines.append(f"{Colors.CYAN}{key}{Colors.RESET}={value}")
            elif line.strip().startswith('#'):
                formatted_lines.append(f"{Colors.DIM}{line}{Colors.RESET}")
            else:
                formatted_lines.append(line)
        lines = formatted_lines
    else:
        lines = code.split('\n')

    for i, line in enumerate(lines, 1):
        if show_line_numbers:
            print(f"  {Colors.DIM}{i:3d}{Colors.RESET} {line}")
        else:
            print(f"  {line}")
print_separator(char='─', width=76)

Print a minimal separator line

Source code in toolboxv2/utils/clis/cli_printing.py
254
255
256
def print_separator(char: str = "─", width: int = 76):
    """Print a minimal separator line"""
    print(f"{Colors.DIM}{char * width}{Colors.RESET}")
print_status(message, status='info')

Print a minimal status message with icon and color

Source code in toolboxv2/utils/clis/cli_printing.py
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def print_status(message: str, status: str = "info"):
    """Print a minimal status message with icon and color"""
    status_config = {
        'success': {'icon': '✓', 'color': Colors.GREEN},
        'error': {'icon': '✗', 'color': Colors.RED},
        'warning': {'icon': '⚠', 'color': Colors.YELLOW},
        'info': {'icon': 'ℹ', 'color': Colors.BLUE},
        'progress': {'icon': '⟳', 'color': Colors.CYAN},
        'waiting': {'icon': '⏳', 'color': Colors.MAGENTA},
        'launch': {'icon': '🚀', 'color': Colors.GREEN},
        'install': {'icon': '📦', 'color': Colors.CYAN},
        'download': {'icon': '⬇️', 'color': Colors.BLUE},
        'upload': {'icon': '⬆️', 'color': Colors.MAGENTA},
        'connect': {'icon': '🔗', 'color': Colors.GREEN},
        'disconnect': {'icon': '🔌', 'color': Colors.RED},
        'configure': {'icon': '🔧', 'color': Colors.YELLOW},
        'debug': {'icon': '🐞', 'color': Colors.MAGENTA},
        'test': {'icon': '🧪', 'color': Colors.GREEN},
        'analyze': {'icon': '🔍', 'color': Colors.BLUE},
        'data': {'icon': '💾', 'color': Colors.YELLOW},
        'database': {'icon': '🗃️', 'color': Colors.MAGENTA},
        'server': {'icon': '🖥️', 'color': Colors.GREEN},
        'network': {'icon': '🌐', 'color': Colors.BLUE},
        'build': {'icon': '🔨', 'color': Colors.CYAN},
        'update': {'icon': '🔄', 'color': Colors.MAGENTA}
    }

    config = status_config.get(status, {'icon': '•', 'color': ''})

    print(f"{config['color']}{config['icon']}{Colors.RESET} {message}")
print_table_header(columns, widths)

Print a table header with columns

Source code in toolboxv2/utils/clis/cli_printing.py
261
262
263
264
265
266
267
268
269
270
def print_table_header(columns: list, widths: list):
    """Print a table header with columns"""
    header_parts = []
    for (name, _), width in zip(columns, widths):
        header_parts.append(f"{Colors.BOLD}{Colors.BRIGHT_WHITE}{name:<{width}}{Colors.RESET}")

    print(f"  {' │ '.join(header_parts)}")

    sep_parts = [f"{Colors.BRIGHT_CYAN}{'─' * w}{Colors.RESET}" for w in widths]
    print(f"  {f'{Colors.BRIGHT_CYAN}─┼─{Colors.RESET}'.join(sep_parts)}")
print_table_row(values, widths, styles=None)

Print a table row

Source code in toolboxv2/utils/clis/cli_printing.py
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
def print_table_row(values: list, widths: list, styles: list = None):
    """Print a table row"""
    if styles is None:
        styles = [""] * len(values)

    color_map = {
        'grey': Colors.GREY,
        'white': Colors.WHITE,
        'green': Colors.BRIGHT_GREEN,
        'yellow': Colors.BRIGHT_YELLOW,
        'cyan': Colors.BRIGHT_CYAN,
        'blue': Colors.BRIGHT_BLUE,
        'red': Colors.BRIGHT_RED,
        'magenta': Colors.BRIGHT_MAGENTA,
    }

    row_parts = []
    for value, width, style in zip(values, widths, styles):
        color = color_map.get(style.lower(), '')
        if color:
            colored_value = f"{color}{value}{Colors.RESET}"
            padding = width - len(value)
            row_parts.append(colored_value + " " * padding)
        else:
            row_parts.append(f"{value:<{width}}")

    print(f"  {f' {Colors.DIM}{Colors.RESET} '.join(row_parts)}")
run_visual_test()

Visual test for all UI components - for alignment and testing

Source code in toolboxv2/utils/clis/cli_printing.py
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
def run_visual_test():
    """Visual test for all UI components - for alignment and testing"""
    print("\n" + f"{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}")
    print(f"{Colors.BOLD}{Colors.BRIGHT_WHITE} VISUAL TEST - CLI UI COMPONENTS ".center(80, '='))
    print(f"{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}\n")

    # Test 1: Headers with different icons
    print(f"{Colors.BOLD}TEST 1: Headers with Different Icons{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")

    print_box_header("Standard Info Header", "ℹ")
    print_box_footer()

    print_box_header("Success Header", "✓")
    print_box_footer()

    print_box_header("Server Header", "🖥️")
    print_box_footer()

    # Test 2: Content with different styles
    print(f"\n{Colors.BOLD}TEST 2: Content with Different Styles{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_box_header("Content Styles Test", "🎨")
    print_box_content("This is a plain text without style")
    print_box_content("This is a success message", "success")
    print_box_content("This is an error message", "error")
    print_box_content("This is a warning message", "warning")
    print_box_content("This is an info message", "info")
    print_box_footer()

    # Test 3: Combined content
    print(f"\n{Colors.BOLD}TEST 3: Combined Content{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_box_header("Server Status Example", "🖥️")
    print_box_content("Server Name: ToolBoxV2 API Server", "info")
    print_box_content("Status: Running", "success")
    print_box_content("Port: 8080", "info")
    print_box_content("Warning: High memory usage detected", "warning")
    print_box_content("Error: Connection timeout on endpoint /api/test", "error")
    print_box_content("Plain text information line")
    print_box_footer()

    # Test 4: Status messages
    print(f"\n{Colors.BOLD}TEST 4: Status Messages{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_status("Success status message", "success")
    print_status("Error status message", "error")
    print_status("Warning status message", "warning")
    print_status("Info status message", "info")
    print_status("Progress status message", "progress")
    print_status("Server status message", "server")
    print_status("Build status message", "build")
    print_status("Update status message", "update")

    # Test 5: Separators
    print(f"\n{Colors.BOLD}TEST 5: Separators{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_separator("─")
    print_separator("═")
    print_separator("━")

    # Test 6: Tables
    print(f"\n{Colors.BOLD}TEST 6: Table Display{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    columns = [
        ("Property", 20),
        ("Value", 30),
        ("Status", 20)
    ]
    widths = [w for _, w in columns]

    print_table_header(columns, widths)
    print_table_row(["Server Name", "ToolBoxV2 API", "Active"], widths, ["white", "cyan", "green"])
    print_table_row(["PID", "12345", "Running"], widths, ["white", "grey", "green"])
    print_table_row(["Version", "1.0.0", "Latest"], widths, ["white", "yellow", "green"])
    print_table_row(["Port", "8080", "Open"], widths, ["white", "blue", "green"])

    # Test 7: Code blocks
    print(f"\n\n{Colors.BOLD}TEST 7: Code & Config File Display{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")

    print_box_header("JSON Configuration", "📄")
    json_example = '''{
  "server": {
    "host": "0.0.0.0",
    "port": 8080,
    "debug": true
  },
  "database": {
    "url": "postgresql://localhost/mydb",
    "pool_size": 10
  }
}'''
    print_code_block(json_example, "json", show_line_numbers=True)
    print_box_footer()

    print_box_header("Environment Variables", "📄")
    env_example = '''# Application Settings
APP_NAME=ToolBoxV2
APP_ENV=production
DEBUG=false

# Database
DATABASE_URL=postgresql://localhost/mydb'''
    print_code_block(env_example, "env")
    print_box_footer()

    # Test 8: Real-world example
    print(f"\n{Colors.BOLD}TEST 8: Real-World Server Start Example{Colors.RESET}")
    print(f"{Colors.DIM}{'-' * 80}{Colors.RESET}")
    print_box_header("Starting Server v1.2.3", "🚀")
    print_box_content("Executable: /usr/local/bin/simple-core-server", "info")
    print_box_content("Host: 0.0.0.0:8080", "info")
    print_box_content("Mode: POSIX Zero-Downtime", "info")
    print_box_footer()

    print_status("Launching server", "progress")
    print_status("Socket created - FD 3 saved to server_socket.fd", "success")
    print()

    print_box_header("Server Started", "✓")
    print_box_content("Version: 1.2.3", "success")
    print_box_content("PID: 54321", "success")
    print_box_content("Port: 8080", "success")
    print_box_footer()

    print(f"\n{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}")
    print(f"{Colors.BOLD}{Colors.BRIGHT_WHITE} END OF VISUAL TEST ".center(80, '='))
    print(f"{Colors.BOLD}{Colors.BRIGHT_CYAN}{'=' * 80}{Colors.RESET}\n")
db_cli_manager
ClusterManager

Manages a cluster of r_blob_db instances defined in a config file.

Source code in toolboxv2/utils/clis/db_cli_manager.py
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
class ClusterManager:
    """Manages a cluster of r_blob_db instances defined in a config file."""

    def __init__(self, config_path: str = CLUSTER_CONFIG_FILE):
        self.config_path = Path(config_path)
        self.instances: dict[str, DBInstanceManager] = self._load_config()

    def _load_config(self) -> dict[str, DBInstanceManager]:
        """Loads and validates the cluster configuration."""
        from toolboxv2 import tb_root_dir
        if not self.config_path.is_absolute():
            self.config_path = tb_root_dir / self.config_path

        default_config_dir = (tb_root_dir / ".data/db_data/").resolve()
        default_config = {
            "instance-01": {"port": 3001, "data_dir": str(default_config_dir / "01")},
            "instance-02": {"port": 3002, "data_dir": str(default_config_dir / "02")},
        }

        if not self.config_path.exists():
            print_status(f"Cluster config '{self.config_path}' not found. Creating default", "warning")

            with open(self.config_path, 'w') as f:
                json.dump(default_config, f, indent=4)
            config_data = default_config
        else:
            try:
                with open(self.config_path) as f:
                    config_data = json.load(f)
            except json.JSONDecodeError:
                print_status(f"Cluster config '{self.config_path}' is not valid JSON. Using default", "error")
                config_data = default_config

        return {id: DBInstanceManager(id, cfg) for id, cfg in config_data.items()}

    def get_instances(self, instance_id: str | None = None) -> list[DBInstanceManager]:
        """Returns a list of instances to operate on."""
        if instance_id:
            if instance_id not in self.instances:
                raise ValueError(f"Instance ID '{instance_id}' not found in '{self.config_path}'.")
            return [self.instances[instance_id]]
        return list(self.instances.values())

    def start_all(self, executable_path: Path, version: str, instance_id: str | None = None):
        """Start all instances"""
        instances = self.get_instances(instance_id)

        if len(instances) > 1:
            print_box_header("Starting Multiple Instances", "🚀")
            print_box_content(f"Total instances: {len(instances)}", "info")
            print_box_footer()

        for instance in instances:
            instance.start(executable_path, version)
            if len(instances) > 1:
                print()

    def stop_all(self, instance_id: str | None = None):
        """Stop all instances"""
        instances = self.get_instances(instance_id)

        if len(instances) > 1:
            print_box_header("Stopping Multiple Instances", "⏹️")
            print_box_content(f"Total instances: {len(instances)}", "info")
            print_box_footer()

        for instance in instances:
            instance.stop()
            if len(instances) > 1:
                print()

    def status_all(self, instance_id: str | None = None, silent=False):
        """Show status of all instances"""
        instances = self.get_instances(instance_id)

        if not silent:

            # Table header
            columns = [
                ("INSTANCE ID", 18),
                ("STATUS", 12),
                ("PID", 8),
                ("VERSION", 12),
                ("PORT", 6),
                ("HOST", 15)
            ]
            widths = [w for _, w in columns]

            print("🖥️ Cluster Status\n")
            print_table_header(columns, widths)

        services_online = 0
        server_list = []

        for instance in instances:
            pid, version = instance.read_state()
            is_running = instance.is_running()

            if is_running:
                server_list.append(f"http://{instance.host}:{instance.port}")
                services_online += 1

            if not silent:
                status_str = " RUNNING" if is_running else " STOPPED"
                status_style = "green" if is_running else "red"

                print_table_row(
                    [
                        instance.id,
                        status_str,
                        str(pid or 'N/A'),
                        version or 'N/A',
                        str(instance.port),
                        instance.host
                    ],
                    widths,
                    ["white", status_style, "grey", "blue", "yellow", "cyan"]
                )

        if not silent:
            print()
            print_status(f"Services online: {services_online}/{len(instances)}", "info")

        return services_online, server_list

    def health_check_all(self, instance_id: str | None = None):
        """Perform health check on all instances"""
        instances = self.get_instances(instance_id)

        print("🏥 Cluster Health Check\n")

        columns = [
            ("INSTANCE ID", 18),
            ("STATUS", 12),
            ("PID", 8),
            ("LATENCY", 10),
            ("BLOBS", 8),
            ("VERSION", 12)
        ]
        widths = [w for _, w in columns]
        print_table_header(columns, widths)

        healthy_count = 0

        for instance in instances:
            health = instance.get_health()
            status = health.get('status', 'UNKNOWN')
            pid = health.get('pid', 'N/A')

            if status == 'OK':
                healthy_count += 1
                status_str, style = " OK", "green"
                latency = f"{health['latency_ms']}ms"
                blobs = str(health.get('blobs_managed', 'N/A'))
                version = health.get('server_version', 'N/A')
            elif status == 'STOPPED':
                status_str, style = "❌ STOPPED", "red"
                latency = blobs = version = "N/A"
            else:
                status_str, style = f"🔥 {status}", "red"
                latency = blobs = version = "N/A"

            print_table_row(
                [instance.id, status_str, str(pid), latency, blobs, version],
                widths,
                ["white", style, "grey", "green", "yellow", "blue"]
            )

        print()
        print_status(f"Healthy instances: {healthy_count}/{len(instances)}",
                     "success" if healthy_count == len(instances) else "warning")

    def update_all_rolling(self, new_executable_path: Path, new_version: str, instance_id: str | None = None):
        """Performs a zero-downtime rolling update of the cluster."""
        instances_to_update = self.get_instances(instance_id)

        print_box_header(f"Rolling Update to Version {new_version}", "🔄")
        print_box_content(f"Instances to update: {len(instances_to_update)}", "info")
        print_box_content(f"Executable: {new_executable_path}", "info")
        print_box_footer()

        for i, instance in enumerate(instances_to_update):
            print_separator("═")
            print(f"  [{i + 1}/{len(instances_to_update)}] Updating instance '{instance.id}'")
            print_separator("═")
            print()

            if not instance.stop():
                print_status(f"CRITICAL: Failed to stop old instance '{instance.id}'. Aborting", "error")
                return

            if not instance.start(new_executable_path, new_version):
                print_status(f"CRITICAL: Failed to start new version for '{instance.id}'", "error")
                print_status("The cluster might be in a partially updated state", "warning")
                return

            # Health check with retries
            print()
            with Spinner(f"Waiting for '{instance.id}' to become healthy", symbols="t") as s:
                for attempt in range(5):
                    s.message = f"Waiting for '{instance.id}' to become healthy (attempt {attempt + 1}/5)"
                    time.sleep(2)
                    health = instance.get_health()
                    if health.get('status') == 'OK':
                        print()
                        print_status(f"Instance '{instance.id}' is healthy with new version", "success")
                        break
                else:
                    print()
                    print_status(f"Instance '{instance.id}' did not become healthy. Update halted", "error")
                    return
            print()

        print_separator("═")
        print_status("Rolling Update Complete!", "success")
        print_separator("═")
get_instances(instance_id=None)

Returns a list of instances to operate on.

Source code in toolboxv2/utils/clis/db_cli_manager.py
406
407
408
409
410
411
412
def get_instances(self, instance_id: str | None = None) -> list[DBInstanceManager]:
    """Returns a list of instances to operate on."""
    if instance_id:
        if instance_id not in self.instances:
            raise ValueError(f"Instance ID '{instance_id}' not found in '{self.config_path}'.")
        return [self.instances[instance_id]]
    return list(self.instances.values())
health_check_all(instance_id=None)

Perform health check on all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
def health_check_all(self, instance_id: str | None = None):
    """Perform health check on all instances"""
    instances = self.get_instances(instance_id)

    print("🏥 Cluster Health Check\n")

    columns = [
        ("INSTANCE ID", 18),
        ("STATUS", 12),
        ("PID", 8),
        ("LATENCY", 10),
        ("BLOBS", 8),
        ("VERSION", 12)
    ]
    widths = [w for _, w in columns]
    print_table_header(columns, widths)

    healthy_count = 0

    for instance in instances:
        health = instance.get_health()
        status = health.get('status', 'UNKNOWN')
        pid = health.get('pid', 'N/A')

        if status == 'OK':
            healthy_count += 1
            status_str, style = " OK", "green"
            latency = f"{health['latency_ms']}ms"
            blobs = str(health.get('blobs_managed', 'N/A'))
            version = health.get('server_version', 'N/A')
        elif status == 'STOPPED':
            status_str, style = "❌ STOPPED", "red"
            latency = blobs = version = "N/A"
        else:
            status_str, style = f"🔥 {status}", "red"
            latency = blobs = version = "N/A"

        print_table_row(
            [instance.id, status_str, str(pid), latency, blobs, version],
            widths,
            ["white", style, "grey", "green", "yellow", "blue"]
        )

    print()
    print_status(f"Healthy instances: {healthy_count}/{len(instances)}",
                 "success" if healthy_count == len(instances) else "warning")
start_all(executable_path, version, instance_id=None)

Start all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py
414
415
416
417
418
419
420
421
422
423
424
425
426
def start_all(self, executable_path: Path, version: str, instance_id: str | None = None):
    """Start all instances"""
    instances = self.get_instances(instance_id)

    if len(instances) > 1:
        print_box_header("Starting Multiple Instances", "🚀")
        print_box_content(f"Total instances: {len(instances)}", "info")
        print_box_footer()

    for instance in instances:
        instance.start(executable_path, version)
        if len(instances) > 1:
            print()
status_all(instance_id=None, silent=False)

Show status of all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
def status_all(self, instance_id: str | None = None, silent=False):
    """Show status of all instances"""
    instances = self.get_instances(instance_id)

    if not silent:

        # Table header
        columns = [
            ("INSTANCE ID", 18),
            ("STATUS", 12),
            ("PID", 8),
            ("VERSION", 12),
            ("PORT", 6),
            ("HOST", 15)
        ]
        widths = [w for _, w in columns]

        print("🖥️ Cluster Status\n")
        print_table_header(columns, widths)

    services_online = 0
    server_list = []

    for instance in instances:
        pid, version = instance.read_state()
        is_running = instance.is_running()

        if is_running:
            server_list.append(f"http://{instance.host}:{instance.port}")
            services_online += 1

        if not silent:
            status_str = " RUNNING" if is_running else " STOPPED"
            status_style = "green" if is_running else "red"

            print_table_row(
                [
                    instance.id,
                    status_str,
                    str(pid or 'N/A'),
                    version or 'N/A',
                    str(instance.port),
                    instance.host
                ],
                widths,
                ["white", status_style, "grey", "blue", "yellow", "cyan"]
            )

    if not silent:
        print()
        print_status(f"Services online: {services_online}/{len(instances)}", "info")

    return services_online, server_list
stop_all(instance_id=None)

Stop all instances

Source code in toolboxv2/utils/clis/db_cli_manager.py
428
429
430
431
432
433
434
435
436
437
438
439
440
def stop_all(self, instance_id: str | None = None):
    """Stop all instances"""
    instances = self.get_instances(instance_id)

    if len(instances) > 1:
        print_box_header("Stopping Multiple Instances", "⏹️")
        print_box_content(f"Total instances: {len(instances)}", "info")
        print_box_footer()

    for instance in instances:
        instance.stop()
        if len(instances) > 1:
            print()
update_all_rolling(new_executable_path, new_version, instance_id=None)

Performs a zero-downtime rolling update of the cluster.

Source code in toolboxv2/utils/clis/db_cli_manager.py
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
def update_all_rolling(self, new_executable_path: Path, new_version: str, instance_id: str | None = None):
    """Performs a zero-downtime rolling update of the cluster."""
    instances_to_update = self.get_instances(instance_id)

    print_box_header(f"Rolling Update to Version {new_version}", "🔄")
    print_box_content(f"Instances to update: {len(instances_to_update)}", "info")
    print_box_content(f"Executable: {new_executable_path}", "info")
    print_box_footer()

    for i, instance in enumerate(instances_to_update):
        print_separator("═")
        print(f"  [{i + 1}/{len(instances_to_update)}] Updating instance '{instance.id}'")
        print_separator("═")
        print()

        if not instance.stop():
            print_status(f"CRITICAL: Failed to stop old instance '{instance.id}'. Aborting", "error")
            return

        if not instance.start(new_executable_path, new_version):
            print_status(f"CRITICAL: Failed to start new version for '{instance.id}'", "error")
            print_status("The cluster might be in a partially updated state", "warning")
            return

        # Health check with retries
        print()
        with Spinner(f"Waiting for '{instance.id}' to become healthy", symbols="t") as s:
            for attempt in range(5):
                s.message = f"Waiting for '{instance.id}' to become healthy (attempt {attempt + 1}/5)"
                time.sleep(2)
                health = instance.get_health()
                if health.get('status') == 'OK':
                    print()
                    print_status(f"Instance '{instance.id}' is healthy with new version", "success")
                    break
            else:
                print()
                print_status(f"Instance '{instance.id}' did not become healthy. Update halted", "error")
                return
        print()

    print_separator("═")
    print_status("Rolling Update Complete!", "success")
    print_separator("═")
DBInstanceManager

Manages a single r_blob_db instance.

Source code in toolboxv2/utils/clis/db_cli_manager.py
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
class DBInstanceManager:
    """Manages a single r_blob_db instance."""

    def __init__(self, instance_id: str, config: dict):
        self.id = instance_id
        self.port = config['port']
        self.host = config.get('host', '127.0.0.1')
        self.data_dir = Path(config['data_dir'])
        self.state_file = self.data_dir / "instance_state.json"
        self.log_file = self.data_dir / "instance.log"

    def read_state(self) -> tuple[int | None, str | None]:
        """Reads the PID and version from the instance's state file."""
        if not self.state_file.exists():
            return None, None
        try:
            with open(self.state_file) as f:
                state = json.load(f)
            return state.get('pid'), state.get('version')
        except (json.JSONDecodeError, ValueError, FileNotFoundError):
            return None, None

    def write_state(self, pid: int | None, version: str | None):
        """Writes the PID and version to the state file."""
        self.data_dir.mkdir(parents=True, exist_ok=True)
        state = {'pid': pid, 'version': version}
        with open(self.state_file, 'w') as f:
            json.dump(state, f, indent=4)

    def is_running(self) -> bool:
        """Checks if the process associated with this instance is running."""
        pid, _ = self.read_state()
        return psutil.pid_exists(pid) if pid else False

    def start(self, executable_path: Path, version: str) -> bool:
        """Starts the instance process and detaches, redirecting output to a log file."""
        if self.is_running():
            print_status(f"Instance '{self.id}' is already running", "warning")
            return True

        print_box_header(f"Starting Instance: {self.id}", "🚀")
        print_box_content(f"Port: {self.port}", "info")
        print_box_content(f"Data Directory: {str(self.data_dir)[:15]}...{str(self.data_dir)[-15:]}", "info")
        print_box_footer()

        self.data_dir.mkdir(parents=True, exist_ok=True)
        log_handle = open(self.log_file, 'a')

        env = os.environ.copy()
        env["R_BLOB_DB_CLEAN"] = os.getenv("R_BLOB_DB_CLEAN", "false")
        env["R_BLOB_DB_PORT"] = str(self.port)
        env["R_BLOB_DB_DATA_DIR"] = str(self.data_dir.resolve())
        env["RUST_LOG"] = "info,tower_http=debug"

        try:
            if executable_path is None:
                raise ValueError(f"Executable not found. Build it first.")

            with Spinner(f"Launching process for '{self.id}'", symbols="d"):
                process = subprocess.Popen(
                    [str(executable_path.resolve())],
                    env=env,
                    stdout=log_handle,
                    stderr=log_handle,
                    creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
                )
                time.sleep(1.5)

            if process.poll() is not None:
                print_status(f"Instance '{self.id}' failed to start. Check logs:", "error")
                print(f"    {Style.GREY(str(self.log_file))}")
                return False

            self.write_state(process.pid, version)
            print_status(f"Instance '{self.id}' started successfully (PID: {process.pid})", "success")
            print_status(f"Logging to: {self.log_file}", "info")
            return True

        except Exception as e:
            print_status(f"Failed to launch instance '{self.id}': {e}", "error")
            log_handle.close()
            return False

    def stop(self, timeout: int = 10) -> bool:
        """Stops the instance process gracefully."""
        if not self.is_running():
            print_status(f"Instance '{self.id}' is not running", "warning")
            self.write_state(None, None)
            return True

        pid, _ = self.read_state()

        print_box_header(f"Stopping Instance: {self.id}", "⏹️")
        print_box_content(f"PID: {pid}", "info")
        print_box_content(f"Timeout: {timeout}s", "info")
        print_box_footer()

        with Spinner(f"Stopping '{self.id}' (PID: {pid})", symbols="+", time_in_s=timeout, count_down=True) as s:
            try:
                proc = psutil.Process(pid)
                proc.terminate()
                proc.wait(timeout)
            except psutil.TimeoutExpired:
                s.message = f"Force killing '{self.id}'"
                proc.kill()
            except psutil.NoSuchProcess:
                pass
            except Exception as e:
                print_status(f"Failed to stop instance '{self.id}': {e}", "error")
                return False

        self.write_state(None, None)
        print_status(f"Instance '{self.id}' stopped", "success")
        return True

    def get_health(self) -> dict:
        """Performs a health check on the running instance."""
        if not self.is_running():
            return {'id': self.id, 'status': 'STOPPED', 'error': 'Process not running'}

        pid, version = self.read_state()
        health_url = f"http://{self.host}:{self.port}/health"
        start_time = time.monotonic()

        try:
            response = requests.get(health_url, timeout=2)
            latency_ms = (time.monotonic() - start_time) * 1000
            response.raise_for_status()
            health_data = response.json()
            health_data.update({
                'id': self.id,
                'pid': pid,
                'latency_ms': round(latency_ms),
                'server_version': health_data.pop('version', 'unknown'),
                'manager_known_version': version
            })
            return health_data
        except requests.exceptions.RequestException as e:
            return {'id': self.id, 'status': 'UNREACHABLE', 'pid': pid, 'error': str(e)}
        except Exception as e:
            return {'id': self.id, 'status': 'ERROR', 'pid': pid, 'error': f'Failed to parse health response: {e}'}

    def get_blob_list(self) -> List[Dict[str, Any]]:
        """Get list of all blobs from this instance"""
        if not self.is_running():
            return []

        # Try API endpoint first
        try:
            response = requests.get(f"http://{self.host}:{self.port}/blobs", timeout=5)
            response.raise_for_status()
            return response.json().get('blobs', [])
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                # Endpoint doesn't exist, fallback to scanning data directory
                print_status("API endpoint /blobs not available, scanning data directory...", "warning")
                return self._scan_data_directory()
            return []
        except Exception as e:
            print_status(f"Error fetching blob list: {e}", "warning")
            # Fallback to scanning data directory
            return self._scan_data_directory()

    def _scan_data_directory(self) -> List[Dict[str, Any]]:
        """Scan the data directory to find blobs (fallback method)"""
        blobs = []

        if not self.data_dir.exists():
            return []

        try:
            # Look for blob files in the data directory
            # Adjust patterns based on your Rust server's storage structure
            for item in self.data_dir.rglob('*'):
                if item.is_file() and not item.name.startswith('.'):
                    # Skip metadata files
                    if item.name in ['instance_state.json', 'instance.log']:
                        continue

                    # Get file stats
                    stat = item.stat()
                    blob_info = {
                        'id': item.stem if item.suffix == '.blob' else item.name,
                        'size': stat.st_size,
                        'created_at': time.strftime('%Y-%m-%d %H:%M:%S',
                                                    time.localtime(stat.st_ctime)),
                        'modified_at': time.strftime('%Y-%m-%d %H:%M:%S',
                                                     time.localtime(stat.st_mtime)),
                        'path': str(item.relative_to(self.data_dir))
                    }
                    blobs.append(blob_info)

            return sorted(blobs, key=lambda x: x['created_at'], reverse=True)

        except Exception as e:
            print_status(f"Error scanning data directory: {e}", "error")
            return []

    def get_blob_data(self, blob_id: str) -> Optional[bytes]:
        """Get data from a specific blob"""
        if not self.is_running():
            # Try to read from disk if server is not running
            return self._read_blob_from_disk(blob_id)

        try:
            response = requests.get(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
            response.raise_for_status()
            return response.content
        except Exception as e:
            print_status(f"Error fetching blob via API: {e}, trying disk...", "warning")
            return self._read_blob_from_disk(blob_id)

    def _read_blob_from_disk(self, blob_id: str) -> Optional[bytes]:
        """Read blob directly from disk (fallback method)"""
        try:
            # Try different possible file patterns
            possible_paths = [
                self.data_dir / blob_id,
                self.data_dir / f"{blob_id}.blob",
                self.data_dir / "blobs" / blob_id,
                self.data_dir / "blobs" / f"{blob_id}.blob",
            ]

            for path in possible_paths:
                if path.exists() and path.is_file():
                    with open(path, 'rb') as f:
                        return f.read()

            return None
        except Exception as e:
            print_status(f"Error reading blob from disk: {e}", "error")
            return None

    def delete_blob(self, blob_id: str) -> bool:
        """Delete a specific blob"""
        if not self.is_running():
            print_status("Instance is not running, cannot delete blob via API", "error")
            return False

        try:
            response = requests.delete(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
            response.raise_for_status()
            return True
        except Exception as e:
            print_status(f"Error deleting blob: {e}", "error")
            return False

    def get_stats(self) -> Dict[str, Any]:
        """Get detailed statistics from this instance"""
        if not self.is_running():
            # Return local stats if server not running
            return self._get_local_stats()

        try:
            # Try stats endpoint
            response = requests.get(f"http://{self.host}:{self.port}/stats", timeout=5)
            response.raise_for_status()
            return response.json()
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                # Stats endpoint doesn't exist, use health endpoint
                health = self.get_health()
                if health.get('status') == 'OK':
                    return {
                        'blob_count': health.get('blobs_managed', 0),
                        'total_size': 'N/A',
                        'status': 'OK'
                    }
            return self._get_local_stats()
        except Exception:
            return self._get_local_stats()

    def _get_local_stats(self) -> Dict[str, Any]:
        """Calculate statistics from local data directory"""
        try:
            if not self.data_dir.exists():
                return {'blob_count': 0, 'total_size': 0}

            total_size = 0
            blob_count = 0

            for item in self.data_dir.rglob('*'):
                if item.is_file() and not item.name.startswith('.'):
                    if item.name not in ['instance_state.json', 'instance.log']:
                        total_size += item.stat().st_size
                        blob_count += 1

            return {
                'blob_count': blob_count,
                'total_size': f"{total_size / 1024:.2f} KB" if total_size < 1024 * 1024
                else f"{total_size / (1024 * 1024):.2f} MB",
                'total_size_bytes': total_size,
                'status': 'offline' if not self.is_running() else 'unknown'
            }
        except Exception as e:
            return {'blob_count': 0, 'total_size': 0, 'error': str(e)}
delete_blob(blob_id)

Delete a specific blob

Source code in toolboxv2/utils/clis/db_cli_manager.py
306
307
308
309
310
311
312
313
314
315
316
317
318
def delete_blob(self, blob_id: str) -> bool:
    """Delete a specific blob"""
    if not self.is_running():
        print_status("Instance is not running, cannot delete blob via API", "error")
        return False

    try:
        response = requests.delete(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
        response.raise_for_status()
        return True
    except Exception as e:
        print_status(f"Error deleting blob: {e}", "error")
        return False
get_blob_data(blob_id)

Get data from a specific blob

Source code in toolboxv2/utils/clis/db_cli_manager.py
271
272
273
274
275
276
277
278
279
280
281
282
283
def get_blob_data(self, blob_id: str) -> Optional[bytes]:
    """Get data from a specific blob"""
    if not self.is_running():
        # Try to read from disk if server is not running
        return self._read_blob_from_disk(blob_id)

    try:
        response = requests.get(f"http://{self.host}:{self.port}/blob/{blob_id}", timeout=5)
        response.raise_for_status()
        return response.content
    except Exception as e:
        print_status(f"Error fetching blob via API: {e}, trying disk...", "warning")
        return self._read_blob_from_disk(blob_id)
get_blob_list()

Get list of all blobs from this instance

Source code in toolboxv2/utils/clis/db_cli_manager.py
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
def get_blob_list(self) -> List[Dict[str, Any]]:
    """Get list of all blobs from this instance"""
    if not self.is_running():
        return []

    # Try API endpoint first
    try:
        response = requests.get(f"http://{self.host}:{self.port}/blobs", timeout=5)
        response.raise_for_status()
        return response.json().get('blobs', [])
    except requests.exceptions.HTTPError as e:
        if e.response.status_code == 404:
            # Endpoint doesn't exist, fallback to scanning data directory
            print_status("API endpoint /blobs not available, scanning data directory...", "warning")
            return self._scan_data_directory()
        return []
    except Exception as e:
        print_status(f"Error fetching blob list: {e}", "warning")
        # Fallback to scanning data directory
        return self._scan_data_directory()
get_health()

Performs a health check on the running instance.

Source code in toolboxv2/utils/clis/db_cli_manager.py
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
def get_health(self) -> dict:
    """Performs a health check on the running instance."""
    if not self.is_running():
        return {'id': self.id, 'status': 'STOPPED', 'error': 'Process not running'}

    pid, version = self.read_state()
    health_url = f"http://{self.host}:{self.port}/health"
    start_time = time.monotonic()

    try:
        response = requests.get(health_url, timeout=2)
        latency_ms = (time.monotonic() - start_time) * 1000
        response.raise_for_status()
        health_data = response.json()
        health_data.update({
            'id': self.id,
            'pid': pid,
            'latency_ms': round(latency_ms),
            'server_version': health_data.pop('version', 'unknown'),
            'manager_known_version': version
        })
        return health_data
    except requests.exceptions.RequestException as e:
        return {'id': self.id, 'status': 'UNREACHABLE', 'pid': pid, 'error': str(e)}
    except Exception as e:
        return {'id': self.id, 'status': 'ERROR', 'pid': pid, 'error': f'Failed to parse health response: {e}'}
get_stats()

Get detailed statistics from this instance

Source code in toolboxv2/utils/clis/db_cli_manager.py
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
def get_stats(self) -> Dict[str, Any]:
    """Get detailed statistics from this instance"""
    if not self.is_running():
        # Return local stats if server not running
        return self._get_local_stats()

    try:
        # Try stats endpoint
        response = requests.get(f"http://{self.host}:{self.port}/stats", timeout=5)
        response.raise_for_status()
        return response.json()
    except requests.exceptions.HTTPError as e:
        if e.response.status_code == 404:
            # Stats endpoint doesn't exist, use health endpoint
            health = self.get_health()
            if health.get('status') == 'OK':
                return {
                    'blob_count': health.get('blobs_managed', 0),
                    'total_size': 'N/A',
                    'status': 'OK'
                }
        return self._get_local_stats()
    except Exception:
        return self._get_local_stats()
is_running()

Checks if the process associated with this instance is running.

Source code in toolboxv2/utils/clis/db_cli_manager.py
102
103
104
105
def is_running(self) -> bool:
    """Checks if the process associated with this instance is running."""
    pid, _ = self.read_state()
    return psutil.pid_exists(pid) if pid else False
read_state()

Reads the PID and version from the instance's state file.

Source code in toolboxv2/utils/clis/db_cli_manager.py
84
85
86
87
88
89
90
91
92
93
def read_state(self) -> tuple[int | None, str | None]:
    """Reads the PID and version from the instance's state file."""
    if not self.state_file.exists():
        return None, None
    try:
        with open(self.state_file) as f:
            state = json.load(f)
        return state.get('pid'), state.get('version')
    except (json.JSONDecodeError, ValueError, FileNotFoundError):
        return None, None
start(executable_path, version)

Starts the instance process and detaches, redirecting output to a log file.

Source code in toolboxv2/utils/clis/db_cli_manager.py
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def start(self, executable_path: Path, version: str) -> bool:
    """Starts the instance process and detaches, redirecting output to a log file."""
    if self.is_running():
        print_status(f"Instance '{self.id}' is already running", "warning")
        return True

    print_box_header(f"Starting Instance: {self.id}", "🚀")
    print_box_content(f"Port: {self.port}", "info")
    print_box_content(f"Data Directory: {str(self.data_dir)[:15]}...{str(self.data_dir)[-15:]}", "info")
    print_box_footer()

    self.data_dir.mkdir(parents=True, exist_ok=True)
    log_handle = open(self.log_file, 'a')

    env = os.environ.copy()
    env["R_BLOB_DB_CLEAN"] = os.getenv("R_BLOB_DB_CLEAN", "false")
    env["R_BLOB_DB_PORT"] = str(self.port)
    env["R_BLOB_DB_DATA_DIR"] = str(self.data_dir.resolve())
    env["RUST_LOG"] = "info,tower_http=debug"

    try:
        if executable_path is None:
            raise ValueError(f"Executable not found. Build it first.")

        with Spinner(f"Launching process for '{self.id}'", symbols="d"):
            process = subprocess.Popen(
                [str(executable_path.resolve())],
                env=env,
                stdout=log_handle,
                stderr=log_handle,
                creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
            )
            time.sleep(1.5)

        if process.poll() is not None:
            print_status(f"Instance '{self.id}' failed to start. Check logs:", "error")
            print(f"    {Style.GREY(str(self.log_file))}")
            return False

        self.write_state(process.pid, version)
        print_status(f"Instance '{self.id}' started successfully (PID: {process.pid})", "success")
        print_status(f"Logging to: {self.log_file}", "info")
        return True

    except Exception as e:
        print_status(f"Failed to launch instance '{self.id}': {e}", "error")
        log_handle.close()
        return False
stop(timeout=10)

Stops the instance process gracefully.

Source code in toolboxv2/utils/clis/db_cli_manager.py
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
def stop(self, timeout: int = 10) -> bool:
    """Stops the instance process gracefully."""
    if not self.is_running():
        print_status(f"Instance '{self.id}' is not running", "warning")
        self.write_state(None, None)
        return True

    pid, _ = self.read_state()

    print_box_header(f"Stopping Instance: {self.id}", "⏹️")
    print_box_content(f"PID: {pid}", "info")
    print_box_content(f"Timeout: {timeout}s", "info")
    print_box_footer()

    with Spinner(f"Stopping '{self.id}' (PID: {pid})", symbols="+", time_in_s=timeout, count_down=True) as s:
        try:
            proc = psutil.Process(pid)
            proc.terminate()
            proc.wait(timeout)
        except psutil.TimeoutExpired:
            s.message = f"Force killing '{self.id}'"
            proc.kill()
        except psutil.NoSuchProcess:
            pass
        except Exception as e:
            print_status(f"Failed to stop instance '{self.id}': {e}", "error")
            return False

    self.write_state(None, None)
    print_status(f"Instance '{self.id}' stopped", "success")
    return True
write_state(pid, version)

Writes the PID and version to the state file.

Source code in toolboxv2/utils/clis/db_cli_manager.py
 95
 96
 97
 98
 99
100
def write_state(self, pid: int | None, version: str | None):
    """Writes the PID and version to the state file."""
    self.data_dir.mkdir(parents=True, exist_ok=True)
    state = {'pid': pid, 'version': version}
    with open(self.state_file, 'w') as f:
        json.dump(state, f, indent=4)
DataDiscovery

Interactive data discovery and manipulation interface

Source code in toolboxv2/utils/clis/db_cli_manager.py
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
class DataDiscovery:
    """Interactive data discovery and manipulation interface"""

    def __init__(self, manager: ClusterManager):
        self.manager = manager
        self.selected_instance: Optional[DBInstanceManager] = None
        self.current_view = "instances"  # instances, blobs, blob_detail
        self.selected_index = 0
        self.blob_list = []
        self.current_blob = None

    async def run(self):
        """Run interactive discovery session"""
        print('\033[2J\033[H')  # Clear screen

        print_box_header("Interactive Data Discovery & Manipulation", "🔍")
        print_box_content("Navigate with ↑↓ or w/s, Enter to select, q to quit", "info")
        print_box_footer()

        while True:
            if self.current_view == "instances":
                action = self.show_instances()
            elif self.current_view == "blobs":
                action = self.show_blobs()
            elif self.current_view == "blob_detail":
                action = self.show_blob_detail()

            if action == "quit":
                break
            elif action == "back":
                self.go_back()

    def show_instances(self):
        """Show instance selection menu with keyboard navigation"""
        import sys

        def get_key():
            """Get single keypress (cross-platform)"""
            if platform.system() == "Windows":
                import msvcrt
                key = msvcrt.getch()
                if key == b'\xe0':  # Arrow key prefix on Windows
                    key = msvcrt.getch()
                    if key == b'H':
                        return 'up'
                    elif key == b'P':
                        return 'down'
                elif key == b'\r':
                    return 'enter'
                elif key in (b'q', b'Q'):
                    return 'quit'
                elif key in (b'w', b'W'):
                    return 'up'
                elif key in (b's', b'S'):
                    return 'down'
                return None
            else:
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                try:
                    tty.setraw(sys.stdin.fileno())
                    ch = sys.stdin.read(1)
                    if ch == '\x1b':  # ESC sequence
                        next_chars = sys.stdin.read(2)
                        if next_chars == '[A':
                            return 'up'
                        elif next_chars == '[B':
                            return 'down'
                    elif ch in ('\r', '\n'):
                        return 'enter'
                    elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                        return 'quit'
                    elif ch in ('w', 'W'):
                        return 'up'
                    elif ch in ('s', 'S'):
                        return 'down'
                    return None
                finally:
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        while True:
            print('\033[2J\033[H')  # Clear screen

            print_box_header("Select Database Instance", "🗄️")
            print()

            instances = self.manager.get_instances()

            for i, instance in enumerate(instances):
                is_selected = i == self.selected_index
                is_running = instance.is_running()

                status_icon = "✅" if is_running else "❌"
                arrow = "▶" if is_selected else " "

                if is_selected:
                    print(f"  {arrow} \033[1;96m{status_icon} {instance.id:<20} Port: {instance.port:<6}\033[0m")
                else:
                    print(f"  {arrow} {status_icon} {instance.id:<20} Port: {instance.port:<6}")

            print()
            print_box_footer()
            print_status("↑↓/w/s: Navigate | Enter: Select | q: Quit", "info")

            # Get single keypress
            key = get_key()

            if key == 'quit':
                return "quit"
            elif key == 'up':
                self.selected_index = max(0, self.selected_index - 1)
            elif key == 'down':
                self.selected_index = min(len(instances) - 1, self.selected_index + 1)
            elif key == 'enter':
                self.selected_instance = instances[self.selected_index]
                if not self.selected_instance.is_running():
                    print()
                    print_status("Instance is not running! Please start it first.", "error")
                    time.sleep(2)
                else:
                    self.current_view = "blobs"
                    self.selected_index = 0
                    self.load_blobs()
                    return "continue"

    def load_blobs(self):
        """Load blob list from selected instance"""
        if not self.selected_instance:
            return

        print_status("Loading blobs...", "progress")
        self.blob_list = self.selected_instance.get_blob_list()

    def show_blobs(self):
        """Show blob list with keyboard navigation"""
        import sys

        def get_key():
            """Get single keypress (cross-platform)"""
            if platform.system() == "Windows":
                import msvcrt
                key = msvcrt.getch()
                if key == b'\xe0':  # Arrow key prefix on Windows
                    key = msvcrt.getch()
                    if key == b'H':
                        return 'up'
                    elif key == b'P':
                        return 'down'
                elif key == b'\r':
                    return 'enter'
                elif key in (b'q', b'Q'):
                    return 'quit'
                elif key in (b'w', b'W'):
                    return 'up'
                elif key in (b's', b'S'):
                    return 'down'
                elif key in (b'd', b'D'):
                    return 'delete'
                elif key in (b'r', b'R'):
                    return 'refresh'
                elif key in (b'b', b'B'):
                    return 'back'
                return None
            else:
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                try:
                    tty.setraw(sys.stdin.fileno())
                    ch = sys.stdin.read(1)
                    if ch == '\x1b':  # ESC sequence
                        next_chars = sys.stdin.read(2)
                        if next_chars == '[A':
                            return 'up'
                        elif next_chars == '[B':
                            return 'down'
                    elif ch in ('\r', '\n'):
                        return 'enter'
                    elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                        return 'quit'
                    elif ch in ('w', 'W'):
                        return 'up'
                    elif ch in ('s', 'S'):
                        return 'down'
                    elif ch in ('d', 'D'):
                        return 'delete'
                    elif ch in ('r', 'R'):
                        return 'refresh'
                    elif ch in ('b', 'B'):
                        return 'back'
                    return None
                finally:
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        while True:
            print('\033[2J\033[H')  # Clear screen

            print_box_header(f"Blobs in {self.selected_instance.id}", "📦")
            print()

            if not self.blob_list:
                print_box_content("No blobs found in this instance", "warning")
                print_box_footer()
                print_status("Press 'b' to go back or 'r' to refresh...", "info")

                key = get_key()
                if key in ('back', 'quit'):
                    return "back"
                elif key == 'refresh':
                    self.load_blobs()
                continue

            # Show stats
            stats = self.selected_instance.get_stats()
            if stats:
                print(f"  Total Blobs: {len(self.blob_list)}")
                if 'total_size' in stats:
                    print(f"  Total Size: {stats.get('total_size', 'N/A')}")
                print()

            # Show blob list
            print_separator()

            # Calculate visible range (show 15 items at a time)
            visible_count = 15
            start_idx = max(0, self.selected_index - visible_count // 2)
            end_idx = min(len(self.blob_list), start_idx + visible_count)

            # Adjust start if we're near the end
            if end_idx - start_idx < visible_count:
                start_idx = max(0, end_idx - visible_count)

            for i in range(start_idx, end_idx):
                blob = self.blob_list[i]
                is_selected = i == self.selected_index
                arrow = "▶" if is_selected else " "

                blob_id = blob.get('id', 'N/A')
                blob_id_display = (blob_id[:37] + '...') if len(blob_id) > 40 else blob_id
                blob_size = blob.get('size', 0)
                size_str = self.format_size(blob_size)

                if is_selected:
                    print(f"  {arrow} \033[1;96m{blob_id_display:<42} {size_str:>10}\033[0m")
                else:
                    print(f"  {arrow} {blob_id_display:<42} {size_str:>10}")

            if len(self.blob_list) > visible_count:
                print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.blob_list)}")

            print()
            print_box_footer()
            print_status("↑↓/w/s: Navigate | Enter: View | d: Delete | r: Refresh | b: Back | q: Quit", "info")

            # Get user input
            key = get_key()

            if key == 'quit':
                return "quit"
            elif key == 'back':
                return "back"
            elif key == 'refresh':
                print()
                print_status("Refreshing blob list...", "progress")
                self.load_blobs()
                self.selected_index = min(self.selected_index, len(self.blob_list) - 1)
                time.sleep(0.5)
            elif key == 'up':
                self.selected_index = max(0, self.selected_index - 1)
            elif key == 'down':
                self.selected_index = min(len(self.blob_list) - 1, self.selected_index + 1)
            elif key == 'delete':
                if 0 <= self.selected_index < len(self.blob_list):
                    if self.confirm_delete_blob():
                        self.delete_current_blob()
                        # Adjust selected index if needed
                        if self.selected_index >= len(self.blob_list):
                            self.selected_index = max(0, len(self.blob_list) - 1)
            elif key == 'enter':
                if 0 <= self.selected_index < len(self.blob_list):
                    self.current_blob = self.blob_list[self.selected_index]
                    self.current_view = "blob_detail"
                    return "continue"

    def show_blob_detail(self):
        """Show detailed view of a blob with keyboard navigation"""
        import sys

        def get_key():
            """Get single keypress (cross-platform)"""
            if platform.system() == "Windows":
                import msvcrt
                key = msvcrt.getch()
                if key == b'\xe0':  # Arrow key prefix on Windows
                    key = msvcrt.getch()
                elif key in (b'q', b'Q'):
                    return 'quit'
                elif key in (b'e', b'E'):
                    return 'export'
                elif key in (b'd', b'D'):
                    return 'delete'
                elif key in (b'b', b'B'):
                    return 'back'
                return None
            else:
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                try:
                    tty.setraw(sys.stdin.fileno())
                    ch = sys.stdin.read(1)
                    if ch == '\x1b':  # ESC sequence
                        sys.stdin.read(2)  # Consume arrow key codes
                    elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                        return 'quit'
                    elif ch in ('e', 'E'):
                        return 'export'
                    elif ch in ('d', 'D'):
                        return 'delete'
                    elif ch in ('b', 'B'):
                        return 'back'
                    return None
                finally:
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        while True:
            print('\033[2J\033[H')  # Clear screen

            blob_id = self.current_blob.get('id', 'N/A')

            print_box_header(f"Blob Details", "📄")
            print()
            print(f"  ID: {blob_id}")
            print(f"  Size: {self.format_size(self.current_blob.get('size', 0))}")

            if 'created_at' in self.current_blob:
                print(f"  Created: {self.current_blob.get('created_at', 'N/A')}")
            if 'modified_at' in self.current_blob:
                print(f"  Modified: {self.current_blob.get('modified_at', 'N/A')}")
            if 'path' in self.current_blob:
                print(f"  Path: {self.current_blob.get('path', 'N/A')}")

            print()

            # Try to load and preview data
            print_status("Loading blob data...", "progress")
            data = self.selected_instance.get_blob_data(blob_id)

            if data:
                print()
                print_separator()
                print("  Data Preview (first 500 bytes):")
                print_separator()

                try:
                    # Try to decode as text
                    text_preview = data[:500].decode('utf-8', errors='ignore')
                    # Replace control characters except newlines
                    text_preview = ''.join(
                        char if char == '\n' or (32 <= ord(char) < 127) else '.'
                        for char in text_preview
                    )
                    print(f"\n{text_preview}\n")
                except:
                    # Show hex dump
                    hex_preview = data[:200].hex()
                    # Format as rows of 32 hex chars
                    for i in range(0, len(hex_preview), 32):
                        print(f"  {hex_preview[i:i + 32]}")
                    print()

                print_separator()
            else:
                print()
                print_status("Could not load blob data", "error")

            print()
            print_box_footer()
            print_status("e: Export | d: Delete | b: Back | q: Quit", "info")

            # Get user input
            key = get_key()

            if key == 'quit':
                return "quit"
            elif key == 'back':
                return "back"
            elif key == 'export':
                if data:
                    # Temporarily restore terminal for input
                    if platform.system() != "Windows":
                        import tty
                        import termios
                        fd = sys.stdin.fileno()
                        old_settings = termios.tcgetattr(fd)
                        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

                    self.export_current_blob(data)

                    # Wait for keypress to continue
                    print()
                    print_status("Press any key to continue...", "info")
                    get_key()
                else:
                    print()
                    print_status("No data to export", "error")
                    time.sleep(1)
            elif key == 'delete':
                # Temporarily restore terminal for confirmation input
                if platform.system() != "Windows":
                    import tty
                    import termios
                    fd = sys.stdin.fileno()
                    old_settings = termios.tcgetattr(fd)
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

                if self.confirm_delete_blob():
                    self.delete_current_blob()
                    return "back"

    def confirm_delete_blob(self) -> bool:
        """Confirm blob deletion with user"""
        if self.current_view == "blob_detail":
            blob = self.current_blob
        elif self.current_view == "blobs":
            if 0 <= self.selected_index < len(self.blob_list):
                blob = self.blob_list[self.selected_index]
            else:
                return False
        else:
            return False

        blob_id = blob.get('id', 'N/A')
        blob_id_short = (blob_id[:20] + '...') if len(blob_id) > 23 else blob_id

        print()
        print_status(f"Really delete blob {blob_id_short}?", "warning")
        print("  Type 'yes' to confirm: ", end='', flush=True)

        # Restore normal input temporarily
        if platform.system() != "Windows":
            import tty
            import termios
            import sys
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

        confirm = input().strip().lower()
        return confirm == 'yes'

    def delete_current_blob(self):
        """Delete the currently selected blob"""
        if self.current_view == "blob_detail":
            blob = self.current_blob
        elif self.current_view == "blobs":
            if 0 <= self.selected_index < len(self.blob_list):
                blob = self.blob_list[self.selected_index]
            else:
                return
        else:
            return

        blob_id = blob.get('id', 'N/A')

        print()
        print_status(f"Deleting blob...", "progress")

        if self.selected_instance.delete_blob(blob_id):
            print_status("Blob deleted successfully", "success")
            self.load_blobs()  # Refresh list
        else:
            print_status("Failed to delete blob", "error")

        time.sleep(1)

    def export_current_blob(self, data: bytes):
        """Export blob data to file"""
        print()
        print("  Enter filename to export to: ", end='', flush=True)

        filename = input().strip()

        if filename:
            try:
                with open(filename, 'wb') as f:
                    f.write(data)
                print_status(f"Blob exported to {filename}", "success")
            except Exception as e:
                print_status(f"Export failed: {e}", "error")
        else:
            print_status("Export cancelled", "warning")

    def go_back(self):
        """Navigate back in the view hierarchy"""
        if self.current_view == "blob_detail":
            self.current_view = "blobs"
            self.current_blob = None
        elif self.current_view == "blobs":
            self.current_view = "instances"
            self.selected_instance = None
            self.blob_list = []
            self.selected_index = 0

    @staticmethod
    def format_size(size_bytes: int) -> str:
        """Format byte size to human readable"""
        for unit in ['B', 'KB', 'MB', 'GB']:
            if size_bytes < 1024.0:
                return f"{size_bytes:.1f}{unit}"
            size_bytes /= 1024.0
        return f"{size_bytes:.1f}TB"
confirm_delete_blob()

Confirm blob deletion with user

Source code in toolboxv2/utils/clis/db_cli_manager.py
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
def confirm_delete_blob(self) -> bool:
    """Confirm blob deletion with user"""
    if self.current_view == "blob_detail":
        blob = self.current_blob
    elif self.current_view == "blobs":
        if 0 <= self.selected_index < len(self.blob_list):
            blob = self.blob_list[self.selected_index]
        else:
            return False
    else:
        return False

    blob_id = blob.get('id', 'N/A')
    blob_id_short = (blob_id[:20] + '...') if len(blob_id) > 23 else blob_id

    print()
    print_status(f"Really delete blob {blob_id_short}?", "warning")
    print("  Type 'yes' to confirm: ", end='', flush=True)

    # Restore normal input temporarily
    if platform.system() != "Windows":
        import tty
        import termios
        import sys
        fd = sys.stdin.fileno()
        old_settings = termios.tcgetattr(fd)
        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    confirm = input().strip().lower()
    return confirm == 'yes'
delete_current_blob()

Delete the currently selected blob

Source code in toolboxv2/utils/clis/db_cli_manager.py
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
def delete_current_blob(self):
    """Delete the currently selected blob"""
    if self.current_view == "blob_detail":
        blob = self.current_blob
    elif self.current_view == "blobs":
        if 0 <= self.selected_index < len(self.blob_list):
            blob = self.blob_list[self.selected_index]
        else:
            return
    else:
        return

    blob_id = blob.get('id', 'N/A')

    print()
    print_status(f"Deleting blob...", "progress")

    if self.selected_instance.delete_blob(blob_id):
        print_status("Blob deleted successfully", "success")
        self.load_blobs()  # Refresh list
    else:
        print_status("Failed to delete blob", "error")

    time.sleep(1)
export_current_blob(data)

Export blob data to file

Source code in toolboxv2/utils/clis/db_cli_manager.py
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
def export_current_blob(self, data: bytes):
    """Export blob data to file"""
    print()
    print("  Enter filename to export to: ", end='', flush=True)

    filename = input().strip()

    if filename:
        try:
            with open(filename, 'wb') as f:
                f.write(data)
            print_status(f"Blob exported to {filename}", "success")
        except Exception as e:
            print_status(f"Export failed: {e}", "error")
    else:
        print_status("Export cancelled", "warning")
format_size(size_bytes) staticmethod

Format byte size to human readable

Source code in toolboxv2/utils/clis/db_cli_manager.py
1099
1100
1101
1102
1103
1104
1105
1106
@staticmethod
def format_size(size_bytes: int) -> str:
    """Format byte size to human readable"""
    for unit in ['B', 'KB', 'MB', 'GB']:
        if size_bytes < 1024.0:
            return f"{size_bytes:.1f}{unit}"
        size_bytes /= 1024.0
    return f"{size_bytes:.1f}TB"
go_back()

Navigate back in the view hierarchy

Source code in toolboxv2/utils/clis/db_cli_manager.py
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
def go_back(self):
    """Navigate back in the view hierarchy"""
    if self.current_view == "blob_detail":
        self.current_view = "blobs"
        self.current_blob = None
    elif self.current_view == "blobs":
        self.current_view = "instances"
        self.selected_instance = None
        self.blob_list = []
        self.selected_index = 0
load_blobs()

Load blob list from selected instance

Source code in toolboxv2/utils/clis/db_cli_manager.py
718
719
720
721
722
723
724
def load_blobs(self):
    """Load blob list from selected instance"""
    if not self.selected_instance:
        return

    print_status("Loading blobs...", "progress")
    self.blob_list = self.selected_instance.get_blob_list()
run() async

Run interactive discovery session

Source code in toolboxv2/utils/clis/db_cli_manager.py
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
async def run(self):
    """Run interactive discovery session"""
    print('\033[2J\033[H')  # Clear screen

    print_box_header("Interactive Data Discovery & Manipulation", "🔍")
    print_box_content("Navigate with ↑↓ or w/s, Enter to select, q to quit", "info")
    print_box_footer()

    while True:
        if self.current_view == "instances":
            action = self.show_instances()
        elif self.current_view == "blobs":
            action = self.show_blobs()
        elif self.current_view == "blob_detail":
            action = self.show_blob_detail()

        if action == "quit":
            break
        elif action == "back":
            self.go_back()
show_blob_detail()

Show detailed view of a blob with keyboard navigation

Source code in toolboxv2/utils/clis/db_cli_manager.py
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
def show_blob_detail(self):
    """Show detailed view of a blob with keyboard navigation"""
    import sys

    def get_key():
        """Get single keypress (cross-platform)"""
        if platform.system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix on Windows
                key = msvcrt.getch()
            elif key in (b'q', b'Q'):
                return 'quit'
            elif key in (b'e', b'E'):
                return 'export'
            elif key in (b'd', b'D'):
                return 'delete'
            elif key in (b'b', b'B'):
                return 'back'
            return None
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    sys.stdin.read(2)  # Consume arrow key codes
                elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                    return 'quit'
                elif ch in ('e', 'E'):
                    return 'export'
                elif ch in ('d', 'D'):
                    return 'delete'
                elif ch in ('b', 'B'):
                    return 'back'
                return None
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    while True:
        print('\033[2J\033[H')  # Clear screen

        blob_id = self.current_blob.get('id', 'N/A')

        print_box_header(f"Blob Details", "📄")
        print()
        print(f"  ID: {blob_id}")
        print(f"  Size: {self.format_size(self.current_blob.get('size', 0))}")

        if 'created_at' in self.current_blob:
            print(f"  Created: {self.current_blob.get('created_at', 'N/A')}")
        if 'modified_at' in self.current_blob:
            print(f"  Modified: {self.current_blob.get('modified_at', 'N/A')}")
        if 'path' in self.current_blob:
            print(f"  Path: {self.current_blob.get('path', 'N/A')}")

        print()

        # Try to load and preview data
        print_status("Loading blob data...", "progress")
        data = self.selected_instance.get_blob_data(blob_id)

        if data:
            print()
            print_separator()
            print("  Data Preview (first 500 bytes):")
            print_separator()

            try:
                # Try to decode as text
                text_preview = data[:500].decode('utf-8', errors='ignore')
                # Replace control characters except newlines
                text_preview = ''.join(
                    char if char == '\n' or (32 <= ord(char) < 127) else '.'
                    for char in text_preview
                )
                print(f"\n{text_preview}\n")
            except:
                # Show hex dump
                hex_preview = data[:200].hex()
                # Format as rows of 32 hex chars
                for i in range(0, len(hex_preview), 32):
                    print(f"  {hex_preview[i:i + 32]}")
                print()

            print_separator()
        else:
            print()
            print_status("Could not load blob data", "error")

        print()
        print_box_footer()
        print_status("e: Export | d: Delete | b: Back | q: Quit", "info")

        # Get user input
        key = get_key()

        if key == 'quit':
            return "quit"
        elif key == 'back':
            return "back"
        elif key == 'export':
            if data:
                # Temporarily restore terminal for input
                if platform.system() != "Windows":
                    import tty
                    import termios
                    fd = sys.stdin.fileno()
                    old_settings = termios.tcgetattr(fd)
                    termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

                self.export_current_blob(data)

                # Wait for keypress to continue
                print()
                print_status("Press any key to continue...", "info")
                get_key()
            else:
                print()
                print_status("No data to export", "error")
                time.sleep(1)
        elif key == 'delete':
            # Temporarily restore terminal for confirmation input
            if platform.system() != "Windows":
                import tty
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            if self.confirm_delete_blob():
                self.delete_current_blob()
                return "back"
show_blobs()

Show blob list with keyboard navigation

Source code in toolboxv2/utils/clis/db_cli_manager.py
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
def show_blobs(self):
    """Show blob list with keyboard navigation"""
    import sys

    def get_key():
        """Get single keypress (cross-platform)"""
        if platform.system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix on Windows
                key = msvcrt.getch()
                if key == b'H':
                    return 'up'
                elif key == b'P':
                    return 'down'
            elif key == b'\r':
                return 'enter'
            elif key in (b'q', b'Q'):
                return 'quit'
            elif key in (b'w', b'W'):
                return 'up'
            elif key in (b's', b'S'):
                return 'down'
            elif key in (b'd', b'D'):
                return 'delete'
            elif key in (b'r', b'R'):
                return 'refresh'
            elif key in (b'b', b'B'):
                return 'back'
            return None
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    next_chars = sys.stdin.read(2)
                    if next_chars == '[A':
                        return 'up'
                    elif next_chars == '[B':
                        return 'down'
                elif ch in ('\r', '\n'):
                    return 'enter'
                elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                    return 'quit'
                elif ch in ('w', 'W'):
                    return 'up'
                elif ch in ('s', 'S'):
                    return 'down'
                elif ch in ('d', 'D'):
                    return 'delete'
                elif ch in ('r', 'R'):
                    return 'refresh'
                elif ch in ('b', 'B'):
                    return 'back'
                return None
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    while True:
        print('\033[2J\033[H')  # Clear screen

        print_box_header(f"Blobs in {self.selected_instance.id}", "📦")
        print()

        if not self.blob_list:
            print_box_content("No blobs found in this instance", "warning")
            print_box_footer()
            print_status("Press 'b' to go back or 'r' to refresh...", "info")

            key = get_key()
            if key in ('back', 'quit'):
                return "back"
            elif key == 'refresh':
                self.load_blobs()
            continue

        # Show stats
        stats = self.selected_instance.get_stats()
        if stats:
            print(f"  Total Blobs: {len(self.blob_list)}")
            if 'total_size' in stats:
                print(f"  Total Size: {stats.get('total_size', 'N/A')}")
            print()

        # Show blob list
        print_separator()

        # Calculate visible range (show 15 items at a time)
        visible_count = 15
        start_idx = max(0, self.selected_index - visible_count // 2)
        end_idx = min(len(self.blob_list), start_idx + visible_count)

        # Adjust start if we're near the end
        if end_idx - start_idx < visible_count:
            start_idx = max(0, end_idx - visible_count)

        for i in range(start_idx, end_idx):
            blob = self.blob_list[i]
            is_selected = i == self.selected_index
            arrow = "▶" if is_selected else " "

            blob_id = blob.get('id', 'N/A')
            blob_id_display = (blob_id[:37] + '...') if len(blob_id) > 40 else blob_id
            blob_size = blob.get('size', 0)
            size_str = self.format_size(blob_size)

            if is_selected:
                print(f"  {arrow} \033[1;96m{blob_id_display:<42} {size_str:>10}\033[0m")
            else:
                print(f"  {arrow} {blob_id_display:<42} {size_str:>10}")

        if len(self.blob_list) > visible_count:
            print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.blob_list)}")

        print()
        print_box_footer()
        print_status("↑↓/w/s: Navigate | Enter: View | d: Delete | r: Refresh | b: Back | q: Quit", "info")

        # Get user input
        key = get_key()

        if key == 'quit':
            return "quit"
        elif key == 'back':
            return "back"
        elif key == 'refresh':
            print()
            print_status("Refreshing blob list...", "progress")
            self.load_blobs()
            self.selected_index = min(self.selected_index, len(self.blob_list) - 1)
            time.sleep(0.5)
        elif key == 'up':
            self.selected_index = max(0, self.selected_index - 1)
        elif key == 'down':
            self.selected_index = min(len(self.blob_list) - 1, self.selected_index + 1)
        elif key == 'delete':
            if 0 <= self.selected_index < len(self.blob_list):
                if self.confirm_delete_blob():
                    self.delete_current_blob()
                    # Adjust selected index if needed
                    if self.selected_index >= len(self.blob_list):
                        self.selected_index = max(0, len(self.blob_list) - 1)
        elif key == 'enter':
            if 0 <= self.selected_index < len(self.blob_list):
                self.current_blob = self.blob_list[self.selected_index]
                self.current_view = "blob_detail"
                return "continue"
show_instances()

Show instance selection menu with keyboard navigation

Source code in toolboxv2/utils/clis/db_cli_manager.py
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
def show_instances(self):
    """Show instance selection menu with keyboard navigation"""
    import sys

    def get_key():
        """Get single keypress (cross-platform)"""
        if platform.system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix on Windows
                key = msvcrt.getch()
                if key == b'H':
                    return 'up'
                elif key == b'P':
                    return 'down'
            elif key == b'\r':
                return 'enter'
            elif key in (b'q', b'Q'):
                return 'quit'
            elif key in (b'w', b'W'):
                return 'up'
            elif key in (b's', b'S'):
                return 'down'
            return None
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    next_chars = sys.stdin.read(2)
                    if next_chars == '[A':
                        return 'up'
                    elif next_chars == '[B':
                        return 'down'
                elif ch in ('\r', '\n'):
                    return 'enter'
                elif ch in ('q', 'Q', '\x03'):  # q or Ctrl+C
                    return 'quit'
                elif ch in ('w', 'W'):
                    return 'up'
                elif ch in ('s', 'S'):
                    return 'down'
                return None
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    while True:
        print('\033[2J\033[H')  # Clear screen

        print_box_header("Select Database Instance", "🗄️")
        print()

        instances = self.manager.get_instances()

        for i, instance in enumerate(instances):
            is_selected = i == self.selected_index
            is_running = instance.is_running()

            status_icon = "✅" if is_running else "❌"
            arrow = "▶" if is_selected else " "

            if is_selected:
                print(f"  {arrow} \033[1;96m{status_icon} {instance.id:<20} Port: {instance.port:<6}\033[0m")
            else:
                print(f"  {arrow} {status_icon} {instance.id:<20} Port: {instance.port:<6}")

        print()
        print_box_footer()
        print_status("↑↓/w/s: Navigate | Enter: Select | q: Quit", "info")

        # Get single keypress
        key = get_key()

        if key == 'quit':
            return "quit"
        elif key == 'up':
            self.selected_index = max(0, self.selected_index - 1)
        elif key == 'down':
            self.selected_index = min(len(instances) - 1, self.selected_index + 1)
        elif key == 'enter':
            self.selected_instance = instances[self.selected_index]
            if not self.selected_instance.is_running():
                print()
                print_status("Instance is not running! Please start it first.", "error")
                time.sleep(2)
            else:
                self.current_view = "blobs"
                self.selected_index = 0
                self.load_blobs()
                return "continue"
cli_db_runner() async

The main entry point for the CLI application.

Source code in toolboxv2/utils/clis/db_cli_manager.py
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
async def cli_db_runner():
    """The main entry point for the CLI application."""

    parser = argparse.ArgumentParser(
        description="🗄️  r_blob_db Cluster Manager - Interactive Database Management",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        prog='tb db',
        epilog="""
╔════════════════════════════════════════════════════════════════════════════╗
║                           Command Examples                                 ║
╠════════════════════════════════════════════════════════════════════════════╣
║                                                                            ║
║  Build & Setup:                                                            ║
║    $ tb db build                     # Build Rust executable               ║
║    $ tb db clean                     # Clean build artifacts               ║
║                                                                            ║
║  Instance Management:                                                      ║
║    $ tb db start                     # Start all instances                 ║
║    $ tb db start --instance-id i1    # Start specific instance             ║
║    $ tb db stop                      # Stop all instances                  ║
║    $ tb db status                    # Show instance status                ║
║                                                                            ║
║  Health & Monitoring:                                                      ║
║    $ tb db health                    # Health check all instances          ║
║    $ tb db health --instance-id i1   # Check specific instance             ║
║                                                                            ║
║  Data Discovery:                                                           ║
║    $ tb db discover                  # Interactive data browser            ║
║                                                                            ║
║  Updates:                                                                  ║
║    $ tb db update --version 1.2.0    # Rolling cluster update              ║
║                                                                            ║
╚════════════════════════════════════════════════════════════════════════════╝
        """
    )

    subparsers = parser.add_subparsers(dest="action", required=False, help="Available actions")

    # Define common arguments
    instance_arg = {
        'name_or_flags': ['--instance-id'],
        'type': str,
        'help': 'Target a specific instance ID. If omitted, action applies to the whole cluster.',
        'default': None
    }
    version_arg = {
        'name_or_flags': ['--version'],
        'type': str,
        'help': 'Specify a version string for the executable (e.g., "1.2.0").',
        'default': 'dev'
    }

    # --- Define Commands ---
    p_start = subparsers.add_parser('start', help='Start database instance(s)')
    p_start.add_argument(*instance_arg['name_or_flags'],
                         **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})
    p_start.add_argument(*version_arg['name_or_flags'],
                         **{k: v for k, v in version_arg.items() if k != 'name_or_flags'})

    p_stop = subparsers.add_parser('stop', help='Stop database instance(s)')
    p_stop.add_argument(*instance_arg['name_or_flags'],
                        **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})

    p_status = subparsers.add_parser('status', help='Show running status of instance(s)')
    p_status.add_argument(*instance_arg['name_or_flags'],
                          **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})

    p_health = subparsers.add_parser('health', help='Perform health check on instance(s)')
    p_health.add_argument(*instance_arg['name_or_flags'],
                          **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})

    p_update = subparsers.add_parser('update', help='Perform rolling update on cluster')
    p_update.add_argument(*instance_arg['name_or_flags'],
                          **{k: v for k, v in instance_arg.items() if k != 'name_or_flags'})
    version_arg_update = {**version_arg, 'required': True}
    p_update.add_argument(*version_arg_update['name_or_flags'],
                          **{k: v for k, v in version_arg_update.items() if k != 'name_or_flags'})

    subparsers.add_parser('build', help='Build the Rust executable from source')
    subparsers.add_parser('clean', help='Clean the Rust build artifacts')
    subparsers.add_parser('discover', help='Interactive data discovery and manipulation')

    # --- Execute Command ---
    args = parser.parse_args()

    if args.action == 'build':
        handle_build()
        return
    if args.action == 'clean':
        handle_clean()
        return

    manager = ClusterManager()

    if args.action == 'discover':
        await handle_discover(manager)
        return
    executable_path = None
    if args.action in ['start', 'update']:
        executable_path = get_executable_path(update=(args.action == 'update'))
        if not executable_path:
            print_status(f"Could not find the {EXECUTABLE_NAME} executable", "error")
            print_status("Please build it first with: tb db build", "info")
            return

    if args.action == 'start':
        manager.start_all(executable_path, args.version, args.instance_id)
    elif args.action == 'stop':
        manager.stop_all(args.instance_id)
    elif args.action == 'status':
        manager.status_all(args.instance_id)
    elif args.action == 'health':
        manager.health_check_all(args.instance_id)
    elif args.action == 'update':
        manager.update_all_rolling(executable_path, args.version, args.instance_id)
    else:
        import asyncio
        await handle_discover(manager)
get_executable_path(base_name=EXECUTABLE_NAME, update=False)

Finds the release executable in standard locations.

Source code in toolboxv2/utils/clis/db_cli_manager.py
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def get_executable_path(base_name: str = EXECUTABLE_NAME, update=False) -> Path | None:
    """Finds the release executable in standard locations."""
    name_with_ext = f"{base_name}.exe" if platform.system() == "Windows" else base_name
    from toolboxv2 import tb_root_dir
    search_paths = [
        tb_root_dir / "bin" / name_with_ext,
        tb_root_dir / "r_blob_db" / "target" / "release" / name_with_ext,
    ]
    if update:
        search_paths = search_paths[::-1]
    for path in search_paths:
        if path.is_file():
            return path.resolve()
    return None
handle_build()

Build the Rust project

Source code in toolboxv2/utils/clis/db_cli_manager.py
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
def handle_build():
    """Build the Rust project"""
    print_box_header("Building r_blob_db", "🔨")
    print_box_content("Compiling Rust project in release mode", "info")
    print_box_footer()

    from toolboxv2 import tb_root_dir

    try:
        with Spinner("Compiling with Cargo", symbols='t'):
            a,b = detect_shell()
            result = subprocess.run(
                [a,b,"cargo", "build", "--release", "--package", "r_blob_db"],
                check=True,
                cwd=tb_root_dir / "r_blob_db",
                capture_output=True,
                text=True
            )

        print_status("Build successful!", "success")

        exe_path = get_executable_path()
        if exe_path:
            bin_dir = tb_root_dir / "bin"
            bin_dir.mkdir(exist_ok=True)
            try:
                dest_path = bin_dir / exe_path.name
                shutil.copy(exe_path, dest_path)
                print_status(f"Executable copied to: {dest_path}", "info")
            except Exception as e:
                print_status(f"Warning: Failed to copy to bin: {e}", "warning")
                # Fallback to ubin
                ubin_dir = tb_root_dir / "ubin"
                ubin_dir.mkdir(exist_ok=True)
                dest_path = ubin_dir / exe_path.name
                try:
                    shutil.copy(exe_path, dest_path)
                    print_status(f"Copied to fallback location: {dest_path}", "info")
                except Exception as e_ubin:
                    print_status(f"Error copying to ubin: {e_ubin}", "error")

    except subprocess.CalledProcessError as e:
        print_status("Build failed!", "error")
        print(Style.GREY(e.stderr))
    except FileNotFoundError:
        print_status("Build failed: 'cargo' command not found", "error")
        print_status("Is Rust installed and in your PATH?", "info")
handle_clean()

Clean build artifacts

Source code in toolboxv2/utils/clis/db_cli_manager.py
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
def handle_clean():
    """Clean build artifacts"""
    print_box_header("Cleaning Build Artifacts", "🧹")
    print_box_footer()

    try:
        with Spinner("Running cargo clean", symbols='+'):
            a,b=detect_shell()
            from toolboxv2 import tb_root_dir
            subprocess.run([a,b,"cargo", "clean"], check=True, capture_output=True, cwd=tb_root_dir/"r_blob_db")
        print_status("Clean successful!", "success")
    except Exception as e:
        print_status(f"Clean failed: {e}", "error")
handle_discover(manager) async

Handle interactive data discovery

Source code in toolboxv2/utils/clis/db_cli_manager.py
1175
1176
1177
1178
async def handle_discover(manager: ClusterManager):
    """Handle interactive data discovery"""
    discovery = DataDiscovery(manager)
    await discovery.run()
tb_lang_cli
cli_tbx_main()

Main entry point for TB Language CLI

Source code in toolboxv2/utils/clis/tb_lang_cli.py
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
def cli_tbx_main():
    """Main entry point for TB Language CLI"""
    Copyparser = argparse.ArgumentParser(
        description="🚀 TB Language - Unified Multi-Language Programming Environment",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        prog='tb run',
        epilog="""
╔════════════════════════════════════════════════════════════════════════════╗
║                           Command Examples                                 ║
╠════════════════════════════════════════════════════════════════════════════╣
║                                                                            ║
║  Setup & Build:                                                            ║
║    $ tb run build                    # Build TB Language (native/release)  ║
║    $ tb run build --debug            # Build in debug mode                 ║
║    $ tb run build --target android   # Build for Android (all archs)       ║
║    $ tb run build --target ios       # Build for iOS (all archs)           ║
║    $ tb run build --target windows   # Cross-compile for Windows           ║
║    $ tb run build --target all       # Build for all platforms             ║
║    $ tb run clean                    # Clean build artifacts               ║
║                                                                            ║
║  Running Programs:                                                         ║
║    $ tb run x program.tb           # Run in JIT mode (default)             ║
║    $ tb run x program.tb --mode compiled                                   ║
║    $ tb run x program.tb --mode streaming                                  ║
║                                                                            ║
║  Compilation:                                                              ║
║    $ tb run compile input.tb output  # Compile to native                   ║
║    $ tb run compile app.tb app.wasm --target wasm                          ║
║                                                                            ║
║  Development:                                                              ║
║    $ tb run repl                     # Start interactive REPL              ║
║    $ tb run check program.tb         # Check syntax & types                ║
║    $ tb run examples                 # Browse and run examples             ║
║                                                                            ║
║  Project Management:                                                       ║
║    $ tb run init myproject           # Create new TB project               ║
║    $ tb run info                     # Show system information             ║
║                                                                            ║
║  Nested Tools:                                                             ║
║    $ tb run support [args]           # System support operations           ║
║    $ tb run ide [args]               # Language IDE extension tools        ║
║    $ tb run test [args]              # TB language testing and examples    ║
║                                                                            ║
╚════════════════════════════════════════════════════════════════════════════╝
"""
    )
    Copysubparsers = Copyparser.add_subparsers(dest="command", required=False)

    # Build command
    p_build = Copysubparsers.add_parser('build', help='Build TB Language executable')
    p_build.add_argument('--debug', action='store_true', help='Build in debug mode')
    p_build.add_argument('--target',
                        choices=['native', 'windows', 'linux', 'macos', 'macos-arm',
                                'android', 'ios', 'all'],
                        default='native',
                        help='Build target platform (default: native)')
    p_build.add_argument('--no-export', action='store_true',
                        help='Skip exporting to bin directory')

    # Clean command
    Copysubparsers.add_parser('clean', help='Clean build artifacts')

    # Run command
    p_run = Copysubparsers.add_parser('x', help='Run a TB program')
    p_run.add_argument('file', help='TB program file to run')
    p_run.add_argument('--mode', choices=['compiled', 'jit', 'streaming'],
                       default='jit', help='Execution mode')
    p_run.add_argument('--watch', action='store_true',
                       help='Watch for file changes and re-run')

    # Compile command
    p_compile = Copysubparsers.add_parser('compile', help='Compile TB program')
    p_compile.add_argument('input', help='Input TB file')
    p_compile.add_argument('output', help='Output file')
    p_compile.add_argument('--target', choices=['native', 'wasm', 'library'],
                           default='native', help='Compilation target')

    # REPL command
    Copysubparsers.add_parser('repl', help='Start interactive REPL')

    # Check command
    p_check = Copysubparsers.add_parser('check', help='Check syntax and types')
    p_check.add_argument('file', help='TB file to check')

    # Init command
    p_init = Copysubparsers.add_parser('init', help='Initialize new TB project')
    p_init.add_argument('name', help='Project name')

    # Examples command
    Copysubparsers.add_parser('examples', help='Browse and run examples')

    # Info command
    Copysubparsers.add_parser('info', help='Show system information')

    # System support command
    p_support = Copysubparsers.add_parser('support', help='System support operations')
    p_support.add_argument('support_args', nargs='*', help='Arguments for system support')

    # IDE extension command
    p_ide = Copysubparsers.add_parser('ide', help='Language IDE extension operations')
    p_ide.add_argument('ide_args', nargs='*', help='Arguments for IDE extension')

    # Test examples command
    p_test = Copysubparsers.add_parser('test', help='TB language testing and examples')
    p_test.add_argument('test_args', nargs='*', help='Arguments for testing')
    p_test.add_argument('--verbose', '-v', action='store_true', help='Verbose output')
    p_test.add_argument('--filter', help='Filter tests by name')
    p_test.add_argument('--failed', '-f', action='store_true', help='Run only failed tests')
    args = Copyparser.parse_args()

    # Execute command
    if args.command == 'build':
        success = handle_build(
            release=not args.debug,
            target=args.target,
            export_bin=not args.no_export
        )
    elif args.command == 'clean':
        success = handle_clean()
    elif args.command == 'x':
        success = handle_run(args.file, mode=args.mode, watch=args.watch)
    elif args.command == 'compile':
        success = handle_compile(args.input, args.output, target=args.target)
    elif args.command == 'repl':
        success = handle_repl()
    elif args.command == 'check':
        success = handle_check(args.file)
    elif args.command == 'init':
        success = handle_init(args.name)
    elif args.command == 'examples':
        success = handle_examples()
    elif args.command == 'info':
        handle_info()
        success = True
    elif args.command == 'support':
        success = handle_system_support(args.support_args)
    elif args.command == 'ide':
        success = handle_ide_extension(args.ide_args)
    elif args.command == 'test':
        success = handle_test_examples(args.test_args)
    else:
        # No command provided, show help
        Copyparser.print_help()
        success = True

    sys.exit(0 if success else 1)
detect_shell()

Detect shell for running commands

Source code in toolboxv2/utils/clis/tb_lang_cli.py
105
106
107
108
109
110
def detect_shell():
    """Detect shell for running commands"""
    if platform.system() == "Windows":
        return "powershell", "-Command"
    else:
        return "sh", "-c"
get_executable_path()

Find the compiled TB executable

Source code in toolboxv2/utils/clis/tb_lang_cli.py
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def get_executable_path() -> Optional[Path]:
    """Find the compiled TB executable"""
    tb_root = get_tb_root()
    name_with_ext = f"{EXECUTABLE_NAME}.exe" if platform.system() == "Windows" else EXECUTABLE_NAME

    search_paths = [
        tb_root / "bin" / name_with_ext,
        get_project_dir() / "target" / "release" / name_with_ext,
        get_project_dir() / "target" / "debug" / name_with_ext,
    ]

    for path in search_paths:
        if path.is_file():
            return path.resolve()

    return None
get_project_dir()

Get the TB language project directory

Source code in toolboxv2/utils/clis/tb_lang_cli.py
82
83
84
def get_project_dir() -> Path:
    """Get the TB language project directory"""
    return get_tb_root() / PROJECT_DIR
get_tb_root()

Get the toolbox root directory

Source code in toolboxv2/utils/clis/tb_lang_cli.py
73
74
75
76
77
78
79
def get_tb_root() -> Path:
    """Get the toolbox root directory"""
    try:
        from toolboxv2 import tb_root_dir
        return tb_root_dir
    except ImportError:
        return Path(__file__).parent.parent.parent
handle_build(release=True, target='native', export_bin=True)

Build the TB language executable for various targets

Parameters:

Name Type Description Default
release bool

Build in release mode (default: True)

True
target str

Build target - native, windows, linux, macos, android, ios, all (default: native)

'native'
export_bin bool

Export binaries to bin directory (default: True)

True
Source code in toolboxv2/utils/clis/tb_lang_cli.py
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
def handle_build(release: bool = True, target: str = "native", export_bin: bool = True):
    """
    Build the TB language executable for various targets

    Args:
        release: Build in release mode (default: True)
        target: Build target - native, windows, linux, macos, android, ios, all (default: native)
        export_bin: Export binaries to bin directory (default: True)
    """
    print_box_header("Building TB Language", "🔨")
    print_box_content(f"Mode: {'Release' if release else 'Debug'}", "info")
    print_box_content(f"Target: {target}", "info")
    print_box_footer()

    project_dir = get_project_dir()

    if not project_dir.exists():
        print_status(f"Project directory not found: {project_dir}", "error")
        return False

    # Define target mappings
    desktop_targets = {
        "windows": "x86_64-pc-windows-msvc",
        "linux": "x86_64-unknown-linux-gnu",
        "macos": "x86_64-apple-darwin",
        "macos-arm": "aarch64-apple-darwin",
    }

    mobile_targets = {
        "android": ["aarch64-linux-android", "armv7-linux-androideabi",
                    "i686-linux-android", "x86_64-linux-android"],
        "ios": ["aarch64-apple-ios", "x86_64-apple-ios", "aarch64-apple-ios-sim"],
    }

    try:
        # Handle different target types
        if target == "native":
            # Build for current platform
            return _build_native(project_dir, release, export_bin)

        elif target in ["windows", "linux", "macos", "macos-arm"]:
            # Build for specific desktop platform
            return _build_desktop_target(project_dir, release, desktop_targets[target], export_bin)

        elif target == "android":
            # Build for all Android targets using mobile script
            return _build_mobile_platform(project_dir, release, "android", export_bin)

        elif target == "ios":
            # Build for all iOS targets using mobile script
            return _build_mobile_platform(project_dir, release, "ios", export_bin)

        elif target == "all":
            # Build for all platforms
            return _build_all_platforms(project_dir, release, export_bin)

        else:
            print_status(f"Unknown target: {target}", "error")
            return False

    except FileNotFoundError:
        print_status("Build failed: 'cargo' command not found", "error")
        print_status("Is Rust installed and in your PATH?", "info")
        print_status("Install from: https://rustup.rs", "info")
        return False
    except Exception as e:
        print_status(f"Build failed: {e}", "error")
        return False
handle_check(file_path)

Check a TB program without executing

Source code in toolboxv2/utils/clis/tb_lang_cli.py
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
def handle_check(file_path: str):
    """Check a TB program without executing"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        return False

    if not Path(file_path).exists():
        print_status(f"File not found: {file_path}", "error")
        return False

    try:
        result = subprocess.run([str(exe_path), "check", file_path], check=True)
        return True
    except subprocess.CalledProcessError:
        return False
    except Exception as e:
        print_status(f"Failed to check: {e}", "error")
        return False
handle_clean()

Clean build artifacts

Source code in toolboxv2/utils/clis/tb_lang_cli.py
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
def handle_clean():
    """Clean build artifacts"""
    print_box_header("Cleaning Build Artifacts", "🧹")
    print_box_footer()

    project_dir = get_project_dir()

    try:
        shell, shell_flag = detect_shell()

        with Spinner("Running cargo clean", symbols='+'):
            subprocess.run(
                [shell, shell_flag, "cargo clean"],
                cwd=project_dir,
                capture_output=True,
                check=True
            )

        print_status("Clean successful!", "success")
        return True
    except Exception as e:
        print_status(f"Clean failed: {e}", "error")
        return False
handle_compile(input_file, output_file, target='native')

Compile a TB program

Source code in toolboxv2/utils/clis/tb_lang_cli.py
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
def handle_compile(input_file: str, output_file: str, target: str = "native"):
    """Compile a TB program"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        return False

    if not Path(input_file).exists():
        print_status(f"Input file not found: {input_file}", "error")
        return False

    print_box_header("Compiling TB Program", "⚙️")
    print_box_content(f"Input: {input_file}", "info")
    print_box_content(f"Output: {output_file}", "info")
    print_box_content(f"Target: {target}", "info")
    print_box_footer()

    try:
        cmd = [str(exe_path), "compile", input_file, output_file, "--target", target]

        result = subprocess.run(cmd, check=True)

        print()
        print_status("Compilation successful!", "success")
        return True

    except subprocess.CalledProcessError:
        print()
        print_status("Compilation failed", "error")
        return False
    except Exception as e:
        print_status(f"Failed to compile: {e}", "error")
        return False
handle_examples()

Run example programs

Source code in toolboxv2/utils/clis/tb_lang_cli.py
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
def handle_examples():
    """Run example programs"""
    examples_dir = get_project_dir() / "examples"
    if not examples_dir.exists():
        print_status("Examples directory not found", "error")
        return False

    examples = list(examples_dir.glob("*.tb"))

    if not examples:
        print_status("No example files found", "warning")
        return False

    print_box_header("TB Language Examples", "📚")
    print()

    for i, example in enumerate(examples, 1):
        print(f"  {i}. {example.name}")

    print()
    print_box_footer()

    try:
        choice = input("Select example (number) or 'q' to quit: ").strip()

        if choice.lower() == 'q':
            return True

        idx = int(choice) - 1
        if 0 <= idx < len(examples):
            print()
            return handle_run(str(examples[idx]), mode="jit")
        else:
            print_status("Invalid selection", "error")
            return False

    except ValueError:
        print_status("Invalid input", "error")
        return False
    except KeyboardInterrupt:
        print()
        return True
handle_ide_extension(args)

Handle language IDE extension operations

Source code in toolboxv2/utils/clis/tb_lang_cli.py
363
364
365
def handle_ide_extension(args):
    """Handle language IDE extension operations"""
    return language_ide_extension(args)
handle_info()

Show system information

Source code in toolboxv2/utils/clis/tb_lang_cli.py
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
def handle_info():
    """Show system information"""
    print_box_header("TB Language System Information", "ℹ️")
    print()
    # TB Root
    tb_root = get_tb_root()
    print(f"  TB Root:     {tb_root}")

    # Project directory
    project_dir = get_project_dir()
    print(f"  Project Dir: {project_dir}")
    print(f"  Exists:      {project_dir.exists()}")

    # Executable
    exe_path = get_executable_path()
    if exe_path:
        print(f"  Executable:  {exe_path}")
        print(f"  Exists:      {exe_path.exists()}")
    else:
        print(f"  Executable:  Not found (build first)")

    # Rust toolchain
    print()
    print("  Rust Toolchain:")
    try:
        result = subprocess.run(
            ["rustc", "--version"],
            capture_output=True,
            text=True,
            check=True
        )
        print(f"    {result.stdout.strip()}")

        result = subprocess.run(
            ["cargo", "--version"],
            capture_output=True,
            text=True,
            check=True
        )
        print(f"    {result.stdout.strip()}")
    except FileNotFoundError:
        print(Style.RED("    Rust not found! Install from https://rustup.rs"))
    except subprocess.CalledProcessError:
        print(Style.RED("    Failed to get Rust version"))

    print()
    print_box_footer()
handle_init(project_name)

Initialize a new TB project

Source code in toolboxv2/utils/clis/tb_lang_cli.py
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
def handle_init(project_name: str):
    """Initialize a new TB project"""
    print_box_header(f"Creating TB Project: {project_name}", "📦")
    print_box_footer()

    from toolboxv2 import tb_root_dir, init_cwd

    if init_cwd == tb_root_dir:
        print_status("Cannot create project in TB root directory", "error")
        return False

    project_path = init_cwd / project_name

    if project_path.exists():
        print_status(f"Directory already exists: {project_path}", "error")
        return False

    try:
        # Create directory structure
        project_path.mkdir()
        (project_path / "src").mkdir()
        (project_path / "examples").mkdir()

        # Create main.tb
        main_tb = project_path / "src" / "main.tb"
        main_tb.write_text('''#!tb
@config {
    mode: "jit"
    type_mode: "static"
    optimize: true
}

@shared {
    app_name: "''' + project_name + '''"
}

fn main() {
    echo "Hello from $app_name!"
}

main()
''')

        # Create README
        readme = project_path / "README.md"
        readme.write_text(f'''# {project_name}

A TB Language project.

## Running


```bash
tb run x src/main.tb
Building
bash
tb compile src/main.tb bin/{project_name}
''')
        print_status(f"✓ Created project structure", "success")
        print_status(f"✓ Created src/main.tb", "success")
        print_status(f"✓ Created README.md", "success")
        print()
        print_status(f"Get started with:", "info")
        print(f"  cd {project_name}")
        print(f"  tb run src/main.tb")

        return True

    except Exception as e:
        print_status(f"Failed to create project: {e}", "error")
        return False
handle_repl()

Start TB REPL

Source code in toolboxv2/utils/clis/tb_lang_cli.py
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
def handle_repl():
    """Start TB REPL"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        return False

    try:
        subprocess.run([str(exe_path), "repl"])
        return True
    except KeyboardInterrupt:
        print()
        return True
    except Exception as e:
        print_status(f"Failed to start REPL: {e}", "error")
        return False
handle_run(file_path, mode='jit', watch=False)

Run a TB program

Source code in toolboxv2/utils/clis/tb_lang_cli.py
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
def handle_run(file_path: str, mode: str = "jit", watch: bool = False):
    """Run a TB program"""
    exe_path = get_executable_path()

    if not exe_path:
        print_status("TB executable not found!", "error")
        print_status("Build it first with: tb x build", "info")
        return False

    if not Path(file_path).exists():
        print_status(f"File not found: {file_path}", "error")
        return False

    print_box_header(f"Running TB Program", "🚀")
    print_box_content(f"File: {file_path}", "info")
    print_box_content(f"Mode: {mode}", "info")
    print_box_footer()

    try:
        cmd = [str(exe_path), "run", file_path, "--mode", mode]

        result = subprocess.run(cmd, check=False)

        if result.returncode == 0:
            print()
            print_status("Execution completed successfully", "success")
            return True
        else:
            print()
            print_status(f"Execution failed with code {result.returncode}", "error")
            return False

    except KeyboardInterrupt:
        print()
        print_status("Execution interrupted", "warning")
        return False
    except Exception as e:
        print_status(f"Failed to run: {e}", "error")
        return False
handle_system_support(args)

Handle system support operations

Source code in toolboxv2/utils/clis/tb_lang_cli.py
359
360
361
def handle_system_support(args):
    """Handle system support operations"""
    return system_tbx_support(*args)
handle_test_examples(args)

Handle TB language testing and examples

Source code in toolboxv2/utils/clis/tb_lang_cli.py
367
368
369
def handle_test_examples(args):
    """Handle TB language testing and examples"""
    return test_tbx_examples(args)
tcm_p2p_cli
ChatListener

Background thread to listen for new chat messages.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
class ChatListener(threading.Thread):
    """Background thread to listen for new chat messages."""

    def __init__(self, chat_manager, room_id, password, callback):
        super().__init__(daemon=True)
        self.chat_manager = chat_manager
        self.room_id = room_id
        self.password = password
        self.callback = callback
        self.running = True
        self.last_message_count = 0

    def run(self):
        while self.running:
            try:
                result = self.chat_manager.get_messages(self.room_id, self.password, 50)
                if result.is_ok():
                    messages = result.get()
                    if len(messages) > self.last_message_count:
                        # New messages arrived
                        new_messages = messages[self.last_message_count:]
                        for msg in new_messages:
                            if not msg['is_own']:  # Only show messages from others
                                self.callback(msg)
                        self.last_message_count = len(messages)
            except Exception:
                pass
            time.sleep(1)  # Poll every second

    def stop(self):
        self.running = False
ChatMessage dataclass

Represents a chat message with encryption support.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
@dataclass
class ChatMessage:
    """Represents a chat message with encryption support."""
    sender: str
    content: str
    timestamp: datetime
    room_id: str
    message_type: MessageType = MessageType.TEXT
    encrypted: bool = True
    file_name: Optional[str] = None
    file_size: Optional[int] = None

    def to_dict(self) -> dict:
        return {
            **asdict(self),
            'timestamp': self.timestamp.isoformat(),
            'message_type': self.message_type.value
        }

    @classmethod
    def from_dict(cls, data: dict) -> 'ChatMessage':
        data['timestamp'] = datetime.fromisoformat(data['timestamp'])
        data['message_type'] = MessageType(data['message_type'])
        return cls(**data)
ChatRoom dataclass

Represents a P2P chat room with E2E encryption.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
@dataclass
class ChatRoom:
    """Represents a P2P chat room with E2E encryption."""
    room_id: str
    name: str
    owner: str
    participants: Set[str]
    is_locked: bool
    is_private: bool
    created_at: datetime
    encryption_key: str
    max_participants: int = 10
    voice_enabled: bool = False
    file_transfer_enabled: bool = True

    def to_dict(self) -> dict:
        return {
            **asdict(self),
            'participants': list(self.participants),
            'created_at': self.created_at.isoformat()
        }

    @classmethod
    def from_dict(cls, data: dict) -> 'ChatRoom':
        data['participants'] = set(data['participants'])
        data['created_at'] = datetime.fromisoformat(data['created_at'])
        return cls(**data)
CryptoManager

Handles all E2E encryption operations.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
class CryptoManager:
    """Handles all E2E encryption operations."""

    @staticmethod
    def generate_room_key(room_id: str, password: str) -> bytes:
        """Generate encryption key for room."""
        salt = room_id.encode()
        kdf = PBKDF2HMAC(
            algorithm=hashes.SHA256(),
            length=32,
            salt=salt,
            iterations=100000,
        )
        return base64.urlsafe_b64encode(kdf.derive(password.encode()))

    @staticmethod
    def encrypt_message(message: str, key: bytes) -> str:
        """Encrypt message content."""
        f = Fernet(key)
        return f.encrypt(message.encode()).decode()

    @staticmethod
    def decrypt_message(encrypted_message: str, key: bytes) -> str:
        """Decrypt message content."""
        f = Fernet(key)
        return f.decrypt(encrypted_message.encode()).decode()

    @staticmethod
    def encrypt_file(file_path: Path, key: bytes) -> bytes:
        """Encrypt file content."""
        f = Fernet(key)
        with open(file_path, 'rb') as file:
            return f.encrypt(file.read())

    @staticmethod
    def decrypt_file(encrypted_data: bytes, key: bytes, output_path: Path):
        """Decrypt file content."""
        f = Fernet(key)
        decrypted_data = f.decrypt(encrypted_data)
        with open(output_path, 'wb') as file:
            file.write(decrypted_data)

    @staticmethod
    def encrypt_bytes(data: bytes, key: bytes) -> bytes:
        """Encrypt binary data directly (for audio/files)."""
        f = Fernet(key)
        return f.encrypt(data)

    @staticmethod
    def decrypt_bytes(encrypted_data: bytes, key: bytes) -> bytes:
        """Decrypt binary data directly (for audio/files)."""
        f = Fernet(key)
        return f.decrypt(encrypted_data)
decrypt_bytes(encrypted_data, key) staticmethod

Decrypt binary data directly (for audio/files).

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
194
195
196
197
198
@staticmethod
def decrypt_bytes(encrypted_data: bytes, key: bytes) -> bytes:
    """Decrypt binary data directly (for audio/files)."""
    f = Fernet(key)
    return f.decrypt(encrypted_data)
decrypt_file(encrypted_data, key, output_path) staticmethod

Decrypt file content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
180
181
182
183
184
185
186
@staticmethod
def decrypt_file(encrypted_data: bytes, key: bytes, output_path: Path):
    """Decrypt file content."""
    f = Fernet(key)
    decrypted_data = f.decrypt(encrypted_data)
    with open(output_path, 'wb') as file:
        file.write(decrypted_data)
decrypt_message(encrypted_message, key) staticmethod

Decrypt message content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
167
168
169
170
171
@staticmethod
def decrypt_message(encrypted_message: str, key: bytes) -> str:
    """Decrypt message content."""
    f = Fernet(key)
    return f.decrypt(encrypted_message.encode()).decode()
encrypt_bytes(data, key) staticmethod

Encrypt binary data directly (for audio/files).

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
188
189
190
191
192
@staticmethod
def encrypt_bytes(data: bytes, key: bytes) -> bytes:
    """Encrypt binary data directly (for audio/files)."""
    f = Fernet(key)
    return f.encrypt(data)
encrypt_file(file_path, key) staticmethod

Encrypt file content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
173
174
175
176
177
178
@staticmethod
def encrypt_file(file_path: Path, key: bytes) -> bytes:
    """Encrypt file content."""
    f = Fernet(key)
    with open(file_path, 'rb') as file:
        return f.encrypt(file.read())
encrypt_message(message, key) staticmethod

Encrypt message content.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
161
162
163
164
165
@staticmethod
def encrypt_message(message: str, key: bytes) -> str:
    """Encrypt message content."""
    f = Fernet(key)
    return f.encrypt(message.encode()).decode()
generate_room_key(room_id, password) staticmethod

Generate encryption key for room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
149
150
151
152
153
154
155
156
157
158
159
@staticmethod
def generate_room_key(room_id: str, password: str) -> bytes:
    """Generate encryption key for room."""
    salt = room_id.encode()
    kdf = PBKDF2HMAC(
        algorithm=hashes.SHA256(),
        length=32,
        salt=salt,
        iterations=100000,
    )
    return base64.urlsafe_b64encode(kdf.derive(password.encode()))
EnhancedInstanceManager

Enhanced instance manager with chat integration.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
class EnhancedInstanceManager:
    """Enhanced instance manager with chat integration."""

    def __init__(self, name: str, app: App):
        self.name = name
        self.app = app
        self.instance_dir = INSTANCES_ROOT_DIR / self.name
        self.state_file = self.instance_dir / "state.json"
        self.config_file = self.instance_dir / "config.toml"
        self.log_file = self.instance_dir / "instance.log"

    def read_state(self) -> dict:
        """Read instance state."""
        if not self.state_file.exists():
            return {}
        try:
            with open(self.state_file) as f:
                return json.load(f)
        except (json.JSONDecodeError, FileNotFoundError):
            return {}

    def write_state(self, state_data: dict):
        """Write instance state."""
        self.instance_dir.mkdir(parents=True, exist_ok=True)
        with open(self.state_file, 'w') as f:
            json.dump(state_data, f, indent=2)

    def is_running(self) -> bool:
        """Check if instance is running."""
        pid = self.read_state().get('pid')
        return psutil.pid_exists(pid) if pid else False

    def generate_config(self, mode: str, config_data: dict):
        """Generate config.toml for instance."""
        content = f'mode = "{mode}"\n\n'

        if mode == "relay":
            content += "[relay]\n"
            content += f'bind_address = "{config_data.get("bind_address", "0.0.0.0:9000")}"\n'
            content += f'password = "{config_data.get("password", "")}"\n'

        elif mode == "peer":
            content += "[peer]\n"
            content += f'relay_address = "{config_data.get("relay_address", "127.0.0.1:9000")}"\n'
            content += f'relay_password = "{config_data.get("relay_password", "")}"\n'
            content += f'peer_id = "{config_data.get("peer_id", "default-peer")}"\n'
            content += f'listen_address = "{config_data.get("listen_address", "127.0.0.1:8000")}"\n'
            content += f'forward_to_address = "{config_data.get("forward_to_address", "127.0.0.1:3000")}"\n'
            if config_data.get("target_peer_id"):
                content += f'target_peer_id = "{config_data.get("target_peer_id")}"\n'

        self.instance_dir.mkdir(parents=True, exist_ok=True)
        with open(self.config_file, "w") as f:
            f.write(content)

    def start(self, executable_path: Path, mode: str, config_data: dict, chat_room: Optional[str] = None) -> bool:
        """Start instance."""
        if self.is_running():
            print(Style.YELLOW(f"Instance '{self.name}' is already running"))
            return True

        self.generate_config(mode, config_data)
        log_handle = open(self.log_file, 'a')

        try:
            with Spinner(f"Starting '{self.name}'", symbols="d"):
                process = subprocess.Popen(
                    [str(executable_path)],
                    cwd=str(self.instance_dir),
                    stdout=log_handle,
                    stderr=log_handle,
                    creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
                )
                time.sleep(1.5)

            if process.poll() is not None:
                print(f"\n{Style.RED2('❌')} Instance failed to start")
                return False

            state = {'pid': process.pid, 'mode': mode, 'config': config_data}
            if chat_room:
                state['chat_room'] = chat_room
            self.write_state(state)

            print(f"\n{Style.GREEN2('✅')} Instance '{Style.Bold(self.name)}' started (PID: {process.pid})")
            if chat_room:
                print(f"   {Style.BLUE('Chat Room:')} {Style.CYAN(chat_room)}")
            return True

        except Exception as e:
            print(f"\n{Style.RED2('❌')} Failed to start: {e}")
            return False

    def stop(self, timeout: int = 10) -> bool:
        """Stop instance."""
        if not self.is_running():
            self.write_state({})
            return True

        pid = self.read_state().get('pid')

        try:
            with Spinner(f"Stopping '{self.name}'", symbols="+", time_in_s=timeout, count_down=True):
                proc = psutil.Process(pid)
                proc.terminate()
                proc.wait(timeout)
        except psutil.TimeoutExpired:
            proc.kill()
        except (psutil.NoSuchProcess, Exception):
            pass

        self.write_state({})
        print(f"\n{Style.VIOLET2('⏹️')} Instance '{Style.Bold(self.name)}' stopped")
        return True
generate_config(mode, config_data)

Generate config.toml for instance.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
def generate_config(self, mode: str, config_data: dict):
    """Generate config.toml for instance."""
    content = f'mode = "{mode}"\n\n'

    if mode == "relay":
        content += "[relay]\n"
        content += f'bind_address = "{config_data.get("bind_address", "0.0.0.0:9000")}"\n'
        content += f'password = "{config_data.get("password", "")}"\n'

    elif mode == "peer":
        content += "[peer]\n"
        content += f'relay_address = "{config_data.get("relay_address", "127.0.0.1:9000")}"\n'
        content += f'relay_password = "{config_data.get("relay_password", "")}"\n'
        content += f'peer_id = "{config_data.get("peer_id", "default-peer")}"\n'
        content += f'listen_address = "{config_data.get("listen_address", "127.0.0.1:8000")}"\n'
        content += f'forward_to_address = "{config_data.get("forward_to_address", "127.0.0.1:3000")}"\n'
        if config_data.get("target_peer_id"):
            content += f'target_peer_id = "{config_data.get("target_peer_id")}"\n'

    self.instance_dir.mkdir(parents=True, exist_ok=True)
    with open(self.config_file, "w") as f:
        f.write(content)
is_running()

Check if instance is running.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1084
1085
1086
1087
def is_running(self) -> bool:
    """Check if instance is running."""
    pid = self.read_state().get('pid')
    return psutil.pid_exists(pid) if pid else False
read_state()

Read instance state.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1068
1069
1070
1071
1072
1073
1074
1075
1076
def read_state(self) -> dict:
    """Read instance state."""
    if not self.state_file.exists():
        return {}
    try:
        with open(self.state_file) as f:
            return json.load(f)
    except (json.JSONDecodeError, FileNotFoundError):
        return {}
start(executable_path, mode, config_data, chat_room=None)

Start instance.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
def start(self, executable_path: Path, mode: str, config_data: dict, chat_room: Optional[str] = None) -> bool:
    """Start instance."""
    if self.is_running():
        print(Style.YELLOW(f"Instance '{self.name}' is already running"))
        return True

    self.generate_config(mode, config_data)
    log_handle = open(self.log_file, 'a')

    try:
        with Spinner(f"Starting '{self.name}'", symbols="d"):
            process = subprocess.Popen(
                [str(executable_path)],
                cwd=str(self.instance_dir),
                stdout=log_handle,
                stderr=log_handle,
                creationflags=subprocess.DETACHED_PROCESS if platform.system() == "Windows" else 0
            )
            time.sleep(1.5)

        if process.poll() is not None:
            print(f"\n{Style.RED2('❌')} Instance failed to start")
            return False

        state = {'pid': process.pid, 'mode': mode, 'config': config_data}
        if chat_room:
            state['chat_room'] = chat_room
        self.write_state(state)

        print(f"\n{Style.GREEN2('✅')} Instance '{Style.Bold(self.name)}' started (PID: {process.pid})")
        if chat_room:
            print(f"   {Style.BLUE('Chat Room:')} {Style.CYAN(chat_room)}")
        return True

    except Exception as e:
        print(f"\n{Style.RED2('❌')} Failed to start: {e}")
        return False
stop(timeout=10)

Stop instance.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
def stop(self, timeout: int = 10) -> bool:
    """Stop instance."""
    if not self.is_running():
        self.write_state({})
        return True

    pid = self.read_state().get('pid')

    try:
        with Spinner(f"Stopping '{self.name}'", symbols="+", time_in_s=timeout, count_down=True):
            proc = psutil.Process(pid)
            proc.terminate()
            proc.wait(timeout)
    except psutil.TimeoutExpired:
        proc.kill()
    except (psutil.NoSuchProcess, Exception):
        pass

    self.write_state({})
    print(f"\n{Style.VIOLET2('⏹️')} Instance '{Style.Bold(self.name)}' stopped")
    return True
write_state(state_data)

Write instance state.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1078
1079
1080
1081
1082
def write_state(self, state_data: dict):
    """Write instance state."""
    self.instance_dir.mkdir(parents=True, exist_ok=True)
    with open(self.state_file, 'w') as f:
        json.dump(state_data, f, indent=2)
FileTransferManager

Manages P2P file transfers with E2E encryption.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
class FileTransferManager:
    """Manages P2P file transfers with E2E encryption."""

    def __init__(self, room_id: str, encryption_key: bytes):
        self.room_id = room_id
        self.encryption_key = encryption_key
        self.transfer_dir = FILE_TRANSFER_DIR / room_id
        self.transfer_dir.mkdir(parents=True, exist_ok=True)

    def prepare_file(self, file_path: Path) -> Tuple[str, int]:
        """Prepare file for transfer (encrypt and chunk)."""
        if not file_path.exists():
            raise FileNotFoundError(f"File not found: {file_path}")

        file_size = file_path.stat().st_size
        if file_size > MAX_FILE_SIZE:
            raise ValueError(f"File too large: {file_size} bytes (max: {MAX_FILE_SIZE})")

        # Encrypt file
        encrypted_data = CryptoManager.encrypt_file(file_path, self.encryption_key)

        # Save encrypted file
        transfer_id = hashlib.sha256(f"{file_path.name}{time.time()}".encode()).hexdigest()[:16]
        encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

        with open(encrypted_file_path, 'wb') as f:
            f.write(encrypted_data)

        return transfer_id, len(encrypted_data)

    def receive_file(self, transfer_id: str, file_name: str) -> Path:
        """Receive and decrypt file."""
        encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

        if not encrypted_file_path.exists():
            raise FileNotFoundError(f"Transfer file not found: {transfer_id}")

        # Read encrypted data
        with open(encrypted_file_path, 'rb') as f:
            encrypted_data = f.read()

        # Decrypt and save
        output_path = FILE_TRANSFER_DIR / "received" / file_name
        output_path.parent.mkdir(parents=True, exist_ok=True)

        CryptoManager.decrypt_file(encrypted_data, self.encryption_key, output_path)

        return output_path
prepare_file(file_path)

Prepare file for transfer (encrypt and chunk).

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
def prepare_file(self, file_path: Path) -> Tuple[str, int]:
    """Prepare file for transfer (encrypt and chunk)."""
    if not file_path.exists():
        raise FileNotFoundError(f"File not found: {file_path}")

    file_size = file_path.stat().st_size
    if file_size > MAX_FILE_SIZE:
        raise ValueError(f"File too large: {file_size} bytes (max: {MAX_FILE_SIZE})")

    # Encrypt file
    encrypted_data = CryptoManager.encrypt_file(file_path, self.encryption_key)

    # Save encrypted file
    transfer_id = hashlib.sha256(f"{file_path.name}{time.time()}".encode()).hexdigest()[:16]
    encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

    with open(encrypted_file_path, 'wb') as f:
        f.write(encrypted_data)

    return transfer_id, len(encrypted_data)
receive_file(transfer_id, file_name)

Receive and decrypt file.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def receive_file(self, transfer_id: str, file_name: str) -> Path:
    """Receive and decrypt file."""
    encrypted_file_path = self.transfer_dir / f"{transfer_id}.enc"

    if not encrypted_file_path.exists():
        raise FileNotFoundError(f"Transfer file not found: {transfer_id}")

    # Read encrypted data
    with open(encrypted_file_path, 'rb') as f:
        encrypted_data = f.read()

    # Decrypt and save
    output_path = FILE_TRANSFER_DIR / "received" / file_name
    output_path.parent.mkdir(parents=True, exist_ok=True)

    CryptoManager.decrypt_file(encrypted_data, self.encryption_key, output_path)

    return output_path
InteractiveP2PCLI

Interactive P2P CLI with modern ToolBox-style interface.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
class InteractiveP2PCLI:
    """Interactive P2P CLI with modern ToolBox-style interface."""

    def __init__(self):
        self.app = get_app("P2P_Interactive_CLI")

        self.voice_server_info: Dict[str, Tuple[str, int]] = {}
        self.chat_manager = P2PChatManager(self.app, self.voice_server_info)
        self.instances: Dict[str, EnhancedInstanceManager] = {}
        self.current_chat_room = None
        self.current_chat_password = None
        self.running = True
        self._load_instances()

        self.file_managers: Dict[str, FileTransferManager] = {}
        self.voice_manager: Optional[VoiceChatManager] = None

    def _load_instances(self):
        """Load existing instances."""
        if INSTANCES_ROOT_DIR.exists():
            for instance_dir in INSTANCES_ROOT_DIR.iterdir():
                if instance_dir.is_dir():
                    self.instances[instance_dir.name] = EnhancedInstanceManager(instance_dir.name, self.app)

    def clear_screen(self):
        """Clear terminal screen."""
        os.system('cls' if os.name == 'nt' else 'clear')

    def print_header(self):
        """Print main header."""
        print(f"""
{Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
{Style.CYAN('║')} {Style.Bold(Style.WHITE('🌐 ToolBox P2P Manager'))} {Style.CYAN('v2.0')} {Style.GREY('- Interactive Mode')} {self._current_room_name() or '':<21} {Style.CYAN('║')}
{Style.CYAN('║')} {Style.GREY('E2E Encrypted Chat • File Transfer • Voice Chat • P2P Tunnels')} {' ' * 6} {Style.CYAN('║')}
{Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}
""")

    def print_menu(self):
        """Print main menu."""
        print(f"""
{Style.Bold(Style.WHITE('┌─ 🎯 MAIN MENU ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('💬 Chat Mode')}          - Start interactive E2E encrypted chat     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('🔧 P2P Configuration')}  - Configure P2P connections                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('📊 Status & Monitoring')} - View connections and rooms              {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('⚙️  Settings')}           - Manage configuration                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('🚪 Exit')}               - Quit application                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

    def chat_menu(self):
        """Interactive chat menu."""
        try:
            while True:
                self.clear_screen()
                self.print_header()

                # Show current room info
                if self.current_chat_room:
                    room = self.chat_manager.rooms.get(self.current_chat_room)
                    if room:
                        print(f"""
    {Style.GREEN('╔══ Current Room ════════════════════════════════════════════════════╗')}
    {Style.GREEN('║')} {Style.WHITE('Name:')} {Style.YELLOW(room.name):<30} {Style.WHITE('ID:')} {Style.CYAN(room.room_id):<15} {" "*22+Style.GREEN('║')}
    {Style.GREEN('║')} {Style.WHITE('Participants:')} {', '.join(list(room.participants)[:10]):<50}{'...' if len(room.participants) > 3 else '':<30}
    {Style.GREEN('╚════════════════════════════════════════════════════════════════════╝')}
    """)

                print(f"""
    {Style.Bold(Style.WHITE('┌─ 💬 CHAT MENU ───────────────────────────────────────────────────────┐'))}
    {Style.WHITE('│')}                                                                      {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Create Room')}         - Create new E2E encrypted chat room         {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Join Room')}           - Join existing room by ID                   {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('List Rooms')}          - Show available chat rooms                  {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Interactive Chat')}    - Start live chat (current room)             {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('5.')} {Style.WHITE('Send File')}           - Transfer file (E2E encrypted)              {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('6.')} {Style.WHITE('Voice Chat')}          - Start voice chat (beta)                    {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('7.')} {Style.WHITE('Lock Room')}           - Lock current room (owner only)             {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('8.')} {Style.WHITE('Leave Room')}          - Leave current chat room                    {Style.WHITE('│')}
    {Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
    {Style.WHITE('│')}                                                                      {Style.WHITE('│')}
    {Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
    """)

                choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

                if choice == '0':
                    break
                elif choice == '1':
                    self._create_chat_room()
                elif choice == '2':
                    self._join_chat_room()
                elif choice == '3':
                    self._list_chat_rooms()
                elif choice == '4':
                    self._interactive_chat()
                elif choice == '5':
                    self._send_file()
                elif choice == '6':
                    self._voice_chat()
                elif choice == '7':
                    self._lock_room()
                elif choice == '8':
                    self._leave_room()
                else:
                    print(f"{Style.RED('Invalid option')}")
                    time.sleep(1)
        finally:
            if self._current_room_name() is not None:
                self._leave_room(auto=True)

    def _create_chat_room(self):
        """Create new chat room."""
        print(f"\n{Style.Bold(Style.CYAN('Create New Chat Room'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Room name:')} ").strip()
        if not name:
            return

        password = input(f"{Style.WHITE('Room password:')} ").strip()
        if not password:
            return

        max_participants = input(f"{Style.WHITE('Max participants (default 10):')} ").strip()
        max_participants = int(max_participants) if max_participants.isdigit() else 10

        voice_enabled = input(f"{Style.WHITE('Enable voice chat? (y/N):')} ").strip().lower() == 'y'
        private = input(f"{Style.WHITE('Make private? (y/N):')} ").strip().lower() == 'y'

        result = self.chat_manager.create_room(name, password, max_participants, voice_enabled, private)

        if result.is_ok():
            data = result.get()
            print(f"\n{Style.GREEN2('✅ Room created successfully!')}")
            print(f"   {Style.WHITE('Room ID:')} {Style.CYAN(data['room_id'])}")
            print(f"   {Style.WHITE('Name:')} {Style.YELLOW(data['name'])}")

            # Auto-join created room
            self.current_chat_room = data['room_id']
            self.current_chat_password = password
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _join_chat_room(self):
        """Join existing chat room."""
        print(f"\n{Style.Bold(Style.CYAN('Join Chat Room'))}")
        print(Style.GREY('─' * 70))

        # First show available rooms
        result = self.chat_manager.list_rooms(show_all=True)
        if result.is_ok():
            rooms = result.get()
            if rooms:
                print(f"\n{Style.WHITE('Available Rooms:')}")
                for i, room in enumerate(rooms, 1):
                    status = "🔒" if room['is_locked'] else "🔓"
                    member = "✓" if room['is_member'] else " "
                    print(
                        f"  {i}. [{member}] {status} {Style.YELLOW(room['name'][:20])} - {Style.CYAN(room['room_id'])}")
                print()

        room_id = input(f"{Style.WHITE('Room ID:')} ").strip()
        if not room_id:
            return

        password = input(f"{Style.WHITE('Password:')} ").strip()
        if not password:
            return

        result = self.chat_manager.join_room(room_id, password)

        if result.is_ok():
            data = result.get()
            self.current_chat_room = room_id
            self.current_chat_password = password

            print(f"\n{Style.GREEN2('✅ Joined room successfully!')}")
            print(f"   {Style.WHITE('Room:')} {Style.YELLOW(data['name'])}")
            print(f"   {Style.WHITE('Participants:')} {', '.join(data['participants'])}")
            if data['voice_enabled']:
                print(f"   {Style.WHITE('Voice chat:')} {Style.GREEN('Enabled')}")
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _list_chat_rooms(self):
        """List available chat rooms."""
        print(f"\n{Style.Bold(Style.CYAN('Chat Rooms'))}")
        print(Style.GREY('═' * 90))

        result = self.chat_manager.list_rooms(show_all=True)

        if result.is_ok():
            rooms = result.get()
            if not rooms:
                print(Style.YELLOW("\n  No chat rooms available"))
            else:
                print(
                    f"\n{Style.Underline('NAME'):<22} {Style.Underline('ROOM ID'):<14} {Style.Underline('OWNER'):<12} {Style.Underline('PARTICIPANTS'):<15} {Style.Underline('STATUS'):<12} {Style.Underline('FEATURES')}")
                print(Style.GREY('─' * 90))

                for room in rooms:
                    name = Style.YELLOW(room['name'][:20])
                    room_id = Style.CYAN(room['room_id'])
                    owner = Style.BLUE(room['owner'][:10])
                    participants = f"{room['participants_count']}/{room['max_participants']}"

                    status_parts = []
                    if room['is_locked']:
                        status_parts.append(Style.RED('🔒 Locked'))
                    if room['is_private']:
                        status_parts.append(Style.YELLOW('🔐 Private'))
                    if not status_parts:
                        status_parts.append(Style.GREEN('🔓 Open'))
                    status = ' '.join(status_parts)[:11]

                    features = []
                    if room['voice_enabled']:
                        features.append('🎤')
                    if room['file_transfer_enabled']:
                        features.append('📁')
                    if room['is_member']:
                        features.append('✓')
                    features_str = ' '.join(features)

                    print(f"{name:<22} {room_id:<14} {owner:<12} {participants:<15} {status:<12} {features_str}")
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _interactive_chat(self):
        """Start interactive chat mode."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room. Join a room first.')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        room = self.chat_manager.rooms.get(self.current_chat_room)
        if not room:
            print(f"{Style.RED2('❌ Room not found')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        self.clear_screen()
        print(f"""
    {Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
    {Style.CYAN('║')} {Style.Bold(Style.WHITE('💬 Interactive Chat'))} - {Style.YELLOW(room.name[:30])} {' ' * (45 - len(room.name[:30]))} {Style.CYAN('║')}
    {Style.CYAN('║')} {Style.GREY('Room ID:')} {Style.CYAN(room.room_id)}{' ' * (59 - len(room.room_id))} {Style.CYAN('║')}
    {Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}

    {Style.GREY('Commands:')} {Style.WHITE('/quit')} - Exit  {Style.WHITE('/file <path>')} - Send file  {Style.WHITE('/refresh')} - Reload messages
    """)
        print(Style.GREY('─' * 70))

        # Show recent messages
        result = self.chat_manager.get_messages(self.current_chat_room, self.current_chat_password, 20)
        message_count = 0
        if result.is_ok():
            messages = result.get()
            message_count = len(messages)
            for msg in messages[-10:]:
                self._display_message(msg)

        print(Style.GREY('─' * 70))

        # Start background listener for new messages
        def on_new_message(msg):
            # Clear current line and display new message
            print(f"\r{' ' * 80}\r", end='')  # Clear line
            self._display_message(msg)
            print(f"{Style.GREEN(f'{self.chat_manager.username}:')} ", end='', flush=True)

        listener = ChatListener(self.chat_manager, self.current_chat_room,
                                self.current_chat_password, on_new_message)
        listener.last_message_count = message_count
        listener.start()

        # Chat loop with non-blocking input
        try:
            while True:
                message = input(f"{Style.GREEN(f'{self.chat_manager.username}:')} ").strip()

                if not message:
                    continue

                if message == '/quit':
                    break

                elif message == '/refresh':
                    # Reload and show recent messages
                    result = self.chat_manager.get_messages(
                        self.current_chat_room,
                        self.current_chat_password,
                        20
                    )
                    if result.is_ok():
                        print(Style.GREY('─' * 70))
                        for msg in result.get()[-10:]:
                            self._display_message(msg)
                        print(Style.GREY('─' * 70))
                        listener.last_message_count = len(result.get())

                elif message.startswith('/file '):
                    file_path = Path(message[6:].strip())
                    self._send_file_inline(file_path)

                elif message == '/voice':
                    print(Style.YELLOW("Voice chat not yet implemented in interactive mode"))

                else:
                    result = self.chat_manager.send_message(
                        self.current_chat_room,
                        message,
                        self.current_chat_password
                    )

                    if result.is_ok():
                        # Display own message
                        self._display_message({
                            'sender': self.chat_manager.username,
                            'content': message,
                            'timestamp': datetime.now().strftime('%H:%M:%S'),
                            'message_type': 'text',
                            'is_own': True
                        })
                        # Update message count to prevent duplicate display
                        listener.last_message_count += 1
                    else:
                        print(f"{Style.RED('❌ Failed to send:')} {result.info}")

        except KeyboardInterrupt:
            pass
        finally:
            listener.stop()
            listener.join(timeout=1)

        print(f"\n{Style.YELLOW('👋 Exiting chat mode')}")
        time.sleep(1)

    def _display_message(self, msg: dict):
        """Display a chat message."""
        timestamp = Style.GREY(f"[{msg['timestamp']}]")

        if msg.get('message_type') == 'system':
            print(f"{timestamp} {Style.VIOLET2('⚙ ')} {Style.GREY(msg['content'])}")
        if msg.get('message_type') == 'file':
            sender_style = Style.GREEN if msg['is_own'] else Style.BLUE
            file_info = f"📁 {msg.get('file_name', 'Unknown')} ({msg.get('file_size', 0)} bytes)"
            sender = sender_style(f'{msg["sender"]}:')
            print(f"{timestamp} {sender} {Style.YELLOW(file_info)}")
        else:
            sender_style = Style.GREEN if msg['is_own'] else Style.BLUE
            sender = sender_style(f'{msg["sender"]}:')
            print(f"{timestamp} {sender} {Style.WHITE(msg['content'])}")

    def _send_file(self):
        """Send file in current room."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        print(f"\n{Style.Bold(Style.CYAN('Send File'))}")
        print(Style.GREY('─' * 70))

        file_path = input(f"{Style.WHITE('File path:')} ").strip()
        if not file_path:
            return

        self._send_file_inline(Path(file_path))
        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _send_file_inline(self, file_path: Path):
        """Send file (internal helper)."""
        if not file_path.exists():
            print(f"{Style.RED2('❌ File not found')}")
            return

        print(f"\n{Style.CYAN('📤 Sending file...')}")

        result = self.chat_manager.send_file(
            self.current_chat_room,
            file_path,
            self.current_chat_password
        )

        if result.is_ok():
            data = result.get()
            print(f"{Style.GREEN2('✅ File sent successfully!')}")
            print(f"   {Style.WHITE('File:')} {data['file_name']}")
            print(f"   {Style.WHITE('Size:')} {data['file_size']} bytes")
        else:
            print(f"{Style.RED2('❌ Failed:')} {result.info}")

    def _voice_chat(self):
        """Start live voice chat with speaker indication."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        room = self.chat_manager.rooms.get(self.current_chat_room)
        if not room or not room.voice_enabled:
            print(f"{Style.RED2('❌ Voice chat not enabled in this room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        if not VOICE_ENABLED:
            print(f"{Style.RED2('❌ pyaudio not installed')}")
            print(f"{Style.YELLOW('Install with:')} pip install pyaudio")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        self.clear_screen()
        print(f"""
    {Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
    {Style.CYAN('║')} {Style.Bold(Style.WHITE('🎤 Live Voice Chat'))} - {Style.YELLOW(room.name[:30])} {' ' * (47 - len(room.name[:30]))} {Style.CYAN('║')}
    {Style.CYAN('║')} {Style.GREY('Press Ctrl+C to exit')} {' ' * 47} {Style.CYAN('║')}
    {Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}
    """)

        try:
            # Initialize voice manager
            key = CryptoManager.generate_room_key(
                self.current_chat_room,
                self.current_chat_password
            )
            voice_mgr = VoiceChatManager(
                self.current_chat_room,
                key,
                self.chat_manager.username
            )

            # Check if we are the host or need to connect
            if self.current_chat_room in self.chat_manager.voice_server_info:
                # Connect to existing voice server
                host, port = self.chat_manager.voice_server_info[self.current_chat_room]
                print(f"{Style.CYAN('🔌 Connecting to voice server...')}")
                voice_mgr.connect_to_voice_server(host, port)
                print(f"{Style.GREEN2('✅ Connected to voice chat!')}\n")
            else:
                # Start as host
                print(f"{Style.CYAN('🎙️  Starting voice server...')}")
                port = voice_mgr.start_voice_server()
                self.chat_manager.voice_server_info[self.current_chat_room] = ('127.0.0.1', port)

                # Also connect to own server
                time.sleep(0.5)
                voice_mgr.connect_to_voice_server('127.0.0.1', port)
                print(f"{Style.GREEN2('✅ Voice server started on port:')} {port}")
                print(f"{Style.YELLOW('Share this info with participants:')}")
                print(f"   Host: 127.0.0.1 (or your public IP)")
                print(f"   Port: {port}\n")

            print(Style.GREY('─' * 70))
            print(f"{Style.WHITE('Voice Chat Active')} - {Style.GREEN('Speak into your microphone')}")
            print(Style.GREY('─' * 70))

            # Start recording thread
            record_thread = threading.Thread(
                target=voice_mgr.start_recording_stream,
                daemon=True
            )
            record_thread.start()

            # Display current speaker in real-time
            last_speaker = None
            print()  # Empty line for speaker display

            try:
                while True:
                    current_speaker = voice_mgr.get_current_speaker()

                    if current_speaker != last_speaker:
                        # Clear previous line and show new speaker
                        print(f"\r{' ' * 70}\r", end='')

                        if current_speaker:
                            if current_speaker == self.chat_manager.username:
                                print(f"\r{Style.GREEN('🎤 You are speaking...')}", end='', flush=True)
                            else:
                                print(f"\r{Style.CYAN(f'🎤 {current_speaker} is speaking...')}", end='', flush=True)
                        else:
                            print(f"\r{Style.GREY('🔇 Silence...')}", end='', flush=True)

                        last_speaker = current_speaker

                    time.sleep(0.1)  # Update display 10 times per second

            except KeyboardInterrupt:
                print(f"\n\n{Style.YELLOW('👋 Exiting voice chat...')}")

        except Exception as e:
            print(f"\n{Style.RED2('❌ Voice chat error:')} {e}")
            import traceback
            traceback.print_exc()

        finally:
            try:
                voice_mgr.cleanup()
            except:
                pass

        print(f"\n{Style.GREEN('Voice chat ended')}")
        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _lock_room(self):
        """Lock current room."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        result = self.chat_manager.lock_room(self.current_chat_room)

        if result.is_ok():
            print(f"\n{Style.GREEN2('✅ Room locked successfully!')}")
        else:
            print(f"\n{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _current_room_name(self):
        """Get name of current room."""
        if not self.current_chat_room:
            return None
        room = self.chat_manager.rooms.get(self.current_chat_room)
        return room.name if room else None

    def _leave_room(self, auto=False):
        """Leave current room."""
        if not self.current_chat_room:
            print(f"{Style.RED2('❌ No active chat room')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        room = self.chat_manager.rooms.get(self.current_chat_room)
        room_name = room.name if room else "Unknown"

        confirm = input(f"\n{Style.YELLOW('⚠ Leave room')} '{room_name}'? (y/N): ").strip().lower() if not auto else 'y'
        if confirm != 'y':
            return

        result = self.chat_manager.leave_room(self.current_chat_room)

        if result.is_ok():
            print(f"\n{Style.GREEN2('✅ Left room successfully')}")
            self.current_chat_room = None
            self.current_chat_password = None
        else:
            print(f"\n{Style.RED2('❌ Failed:')} {result.info}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def p2p_menu(self):
        """P2P configuration menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ 🔧 P2P CONFIGURATION ───────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Start Relay Server')}  - Become a relay for P2P connections         {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Connect as Peer')}     - Connect to relay and other peers           {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Expose Local Service')} - Make local service accessible via P2P     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Stop Instance')}       - Stop a running P2P instance                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._start_relay()
            elif choice == '2':
                self._connect_peer()
            elif choice == '3':
                self._expose_service()
            elif choice == '4':
                self._stop_instance()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)

    def _start_relay(self):
        """Start relay server."""
        print(f"\n{Style.Bold(Style.CYAN('Start Relay Server'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Instance name (default: relay):')} ").strip() or "relay"
        bind = input(f"{Style.WHITE('Bind address (default: 0.0.0.0:9000):')} ").strip() or "0.0.0.0:9000"
        password = input(f"{Style.WHITE('Relay password:')} ").strip()

        if not password:
            print(f"{Style.RED2('❌ Password required')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Get executable path
        executable = self._get_executable_path()
        if not executable:
            print(f"{Style.RED2('❌ Executable not found. Run')} {Style.WHITE('tb p2p build')} {Style.RED2('first')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Create instance
        instance = EnhancedInstanceManager(name, self.app)
        config = {'bind_address': bind, 'password': password}

        success = instance.start(executable, 'relay', config)

        if success:
            self.instances[name] = instance

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _connect_peer(self):
        """Connect as peer."""
        print(f"\n{Style.Bold(Style.CYAN('Connect as Peer'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Instance name:')} ").strip()
        if not name:
            return

        relay_addr = input(f"{Style.WHITE('Relay address (e.g., 127.0.0.1:9000):')} ").strip()
        relay_pass = input(f"{Style.WHITE('Relay password:')} ").strip()
        peer_id = input(f"{Style.WHITE('Your peer ID (default: instance name):')} ").strip() or name
        listen = input(f"{Style.WHITE('Listen address (default: 127.0.0.1:8000):')} ").strip() or "127.0.0.1:8000"
        target = input(f"{Style.WHITE('Target peer ID (optional):')} ").strip()

        if not all([relay_addr, relay_pass]):
            print(f"{Style.RED2('❌ Missing required fields')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Optional: Link to chat room
        link_chat = input(f"{Style.WHITE('Link to chat room? (y/N):')} ").strip().lower() == 'y'
        chat_room = None

        if link_chat and self.current_chat_room:
            chat_room = self.current_chat_room

        # Get executable path
        executable = self._get_executable_path()
        if not executable:
            print(f"{Style.RED2('❌ Executable not found')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Create instance
        instance = EnhancedInstanceManager(name, self.app)
        config = {
            'relay_address': relay_addr,
            'relay_password': relay_pass,
            'peer_id': peer_id,
            'listen_address': listen,
            'target_peer_id': target if target else None
        }

        success = instance.start(executable, 'peer', config, chat_room)

        if success:
            self.instances[name] = instance

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _expose_service(self):
        """Expose local service via P2P."""
        print(f"\n{Style.Bold(Style.CYAN('Expose Local Service'))}")
        print(Style.GREY('─' * 70))

        name = input(f"{Style.WHITE('Instance name:')} ").strip()
        if not name:
            return

        relay_addr = input(f"{Style.WHITE('Relay address:')} ").strip()
        relay_pass = input(f"{Style.WHITE('Relay password:')} ").strip()
        peer_id = input(f"{Style.WHITE('Your peer ID:')} ").strip() or name
        listen = input(f"{Style.WHITE('Listen address (default: 127.0.0.1:8000):')} ").strip() or "127.0.0.1:8000"
        forward = input(f"{Style.WHITE('Forward to (local service, e.g., 127.0.0.1:3000):')} ").strip()

        if not all([relay_addr, relay_pass, forward]):
            print(f"{Style.RED2('❌ Missing required fields')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Get executable path
        executable = self._get_executable_path()
        if not executable:
            print(f"{Style.RED2('❌ Executable not found')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        # Create instance
        instance = EnhancedInstanceManager(name, self.app)
        config = {
            'relay_address': relay_addr,
            'relay_password': relay_pass,
            'peer_id': peer_id,
            'listen_address': listen,
            'forward_to_address': forward
        }

        success = instance.start(executable, 'peer', config)

        if success:
            self.instances[name] = instance

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _stop_instance(self):
        """Stop running instance."""
        if not self.instances:
            print(f"\n{Style.YELLOW('No running instances')}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        print(f"\n{Style.Bold(Style.CYAN('Stop Instance'))}")
        print(Style.GREY('─' * 70))

        print(f"\n{Style.WHITE('Running instances:')}")
        running = {name: inst for name, inst in self.instances.items() if inst.is_running()}

        if not running:
            print(Style.YELLOW("  No running instances"))
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        for i, (name, inst) in enumerate(running.items(), 1):
            state = inst.read_state()
            mode = state.get('mode', 'Unknown')
            pid = state.get('pid', 'N/A')
            print(f"  {i}. {Style.YELLOW(name)} ({mode}, PID: {pid})")

        name = input(f"\n{Style.WHITE('Instance name to stop:')} ").strip()

        if name in running:
            running[name].stop()
        else:
            print(f"{Style.RED2('❌ Instance not found')}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _get_executable_path(self) -> Optional[Path]:
        """Get executable path."""
        search_paths = [
            tb_root_dir / "bin" / EXECUTABLE_NAME,
            tb_root_dir / "tcm" / "target" / "release" / EXECUTABLE_NAME,
        ]

        for path in search_paths:
            if path.is_file():
                return path.resolve()

        return None

    def status_menu(self, do_clear=True):
        """Status and monitoring menu."""
        self.clear_screen() if do_clear else None
        self.print_header() if do_clear else None

        print(f"\n{Style.Bold(Style.CYAN('📊 System Status'))}")
        print(Style.GREY('═' * 90))

        # P2P Instances
        print(f"\n{Style.Bold(Style.WHITE('P2P Instances:'))}")
        if not self.instances:
            print(Style.YELLOW("  No instances configured"))
        else:
            print(
                f"\n{Style.Underline('NAME'):<20} {Style.Underline('MODE'):<12} {Style.Underline('STATUS'):<12} {Style.Underline('PID'):<10} {Style.Underline('CHAT ROOM')}")
            print(Style.GREY('─' * 90))

            for name, inst in self.instances.items():
                state = inst.read_state()
                mode = state.get('mode', 'Unknown')
                pid = state.get('pid', 'N/A')
                chat_room = state.get('chat_room', '-')
                status = Style.GREEN('✅ Running') if inst.is_running() else Style.RED('❌ Stopped')

                print(
                    f"{Style.YELLOW(name):<20} {mode:<12} {status:<12} {str(pid):<10} {Style.CYAN(str(chat_room)[:20])}")

        # Chat Rooms
        print(f"\n{Style.Bold(Style.WHITE('Chat Rooms:'))}")
        result = self.chat_manager.list_rooms()

        if result.is_ok():
            rooms = result.get()
            if not rooms:
                print(Style.YELLOW("  No chat rooms"))
            else:
                print(
                    f"\n{Style.Underline('NAME'):<20} {Style.Underline('PARTICIPANTS'):<15} {Style.Underline('STATUS'):<15} {Style.Underline('FEATURES')}")
                print(Style.GREY('─' * 70))

                for room in rooms:
                    name = Style.YELLOW(room['name'][:18])
                    participants = f"{room['participants_count']}/{room['max_participants']}"

                    status_parts = []
                    if room['is_locked']:
                        status_parts.append('🔒')
                    if room['is_private']:
                        status_parts.append('🔐')
                    if room['is_member']:
                        status_parts.append('✓')
                    status = ' '.join(status_parts) if status_parts else '🔓'

                    features = []
                    if room['voice_enabled']:
                        features.append('🎤 Voice')
                    if room['file_transfer_enabled']:
                        features.append('📁 Files')
                    features_str = ', '.join(features)

                    print(f"{name:<20} {participants:<15} {status:<15} {features_str}")

        input(f"\n{Style.GREY('Press Enter to continue...')}") if do_clear else None

    def settings_menu(self):
        """Settings menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ ⚙️  SETTINGS ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Change Username')}    - Set display name for chat                   {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Build P2P Binary')}   - Compile Rust P2P application                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Clean Up')}           - Remove old instances and data               {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}               - Return to main menu                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._change_username()
            elif choice == '2':
                self._build_binary()
            elif choice == '3':
                self._cleanup()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)

    def _change_username(self):
        """Change username."""
        print(f"\n{Style.Bold(Style.CYAN('Change Username'))}")
        print(Style.GREY('─' * 70))
        print(f"{Style.WHITE('Current:')} {Style.YELLOW(self.chat_manager.username)}")

        new_name = input(f"\n{Style.WHITE('New username:')} ").strip()
        if new_name:
            self.chat_manager.username = new_name
            print(f"\n{Style.GREEN2('✅ Username changed to:')} {Style.YELLOW(new_name)}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _build_binary(self):
        """Build P2P binary."""
        print(f"\n{Style.Bold(Style.CYAN('Building P2P Binary'))}")
        print(Style.GREY('─' * 70))

        tcm_dir = tb_root_dir / "tcm"
        if not tcm_dir.exists():
            print(f"{Style.RED2('❌ TCM directory not found at:')} {tcm_dir}")
            input(f"\n{Style.GREY('Press Enter to continue...')}")
            return

        print(f"\n{Style.CYAN('⚙ Building with Cargo...')}")

        try:
            with Spinner("Compiling Rust project", symbols="t", time_in_s=120):
                process = subprocess.run(
                    ["cargo", "build", "--release"],
                    cwd=str(tcm_dir),
                    capture_output=True,
                    text=True
                )

            if process.returncode == 0:
                print(f"\n{Style.GREEN2('✅ Build successful!')}")

                # Copy to bin directory
                source = tcm_dir / "target" / "release" / EXECUTABLE_NAME
                dest_dir = tb_root_dir / "bin"
                dest_dir.mkdir(exist_ok=True)
                dest = dest_dir / EXECUTABLE_NAME

                if source.exists():
                    import shutil
                    shutil.copy2(source, dest)
                    print(f"{Style.GREEN('📦 Copied to:')} {dest}")
            else:
                print(f"\n{Style.RED2('❌ Build failed:')}")
                print(Style.GREY(process.stderr))

        except FileNotFoundError:
            print(f"\n{Style.RED2('❌ Cargo not found. Is Rust installed?')}")
        except Exception as e:
            print(f"\n{Style.RED2('❌ Build error:')} {e}")

        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def _cleanup(self):
        """Cleanup old data."""
        print(f"\n{Style.Bold(Style.YELLOW('⚠ Cleanup'))}")
        print(Style.GREY('─' * 70))
        print(f"{Style.RED('This will:')}")
        print(f"  • Stop all running instances")
        print(f"  • Delete instance configurations")
        print(f"  • Keep chat rooms and messages")

        confirm = input(f"\n{Style.WHITE('Continue? (y/N):')} ").strip().lower()
        if confirm != 'y':
            return

        # Stop all instances
        for inst in self.instances.values():
            if inst.is_running():
                inst.stop()

        # Remove instance directory
        if INSTANCES_ROOT_DIR.exists():
            import shutil
            shutil.rmtree(INSTANCES_ROOT_DIR)

        self.instances = {}

        print(f"\n{Style.GREEN2('✅ Cleanup complete')}")
        input(f"\n{Style.GREY('Press Enter to continue...')}")

    def run(self):
        """Main application loop."""
        while self.running:
            self.clear_screen()
            self.print_header()
            self.print_menu()

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                print(f"\n{Style.YELLOW('👋 Goodbye!')}")
                self.running = False
            elif choice == '1':
                self.chat_menu()
            elif choice == '2':
                self.p2p_menu()
            elif choice == '3':
                self.status_menu()
            elif choice == '4':
                self.settings_menu()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)
chat_menu()

Interactive chat menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
def chat_menu(self):
    """Interactive chat menu."""
    try:
        while True:
            self.clear_screen()
            self.print_header()

            # Show current room info
            if self.current_chat_room:
                room = self.chat_manager.rooms.get(self.current_chat_room)
                if room:
                    print(f"""
{Style.GREEN('╔══ Current Room ════════════════════════════════════════════════════╗')}
{Style.GREEN('║')} {Style.WHITE('Name:')} {Style.YELLOW(room.name):<30} {Style.WHITE('ID:')} {Style.CYAN(room.room_id):<15} {" "*22+Style.GREEN('║')}
{Style.GREEN('║')} {Style.WHITE('Participants:')} {', '.join(list(room.participants)[:10]):<50}{'...' if len(room.participants) > 3 else '':<30}
{Style.GREEN('╚════════════════════════════════════════════════════════════════════╝')}
""")

            print(f"""
{Style.Bold(Style.WHITE('┌─ 💬 CHAT MENU ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Create Room')}         - Create new E2E encrypted chat room         {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Join Room')}           - Join existing room by ID                   {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('List Rooms')}          - Show available chat rooms                  {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Interactive Chat')}    - Start live chat (current room)             {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('5.')} {Style.WHITE('Send File')}           - Transfer file (E2E encrypted)              {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('6.')} {Style.WHITE('Voice Chat')}          - Start voice chat (beta)                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('7.')} {Style.WHITE('Lock Room')}           - Lock current room (owner only)             {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('8.')} {Style.WHITE('Leave Room')}          - Leave current chat room                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._create_chat_room()
            elif choice == '2':
                self._join_chat_room()
            elif choice == '3':
                self._list_chat_rooms()
            elif choice == '4':
                self._interactive_chat()
            elif choice == '5':
                self._send_file()
            elif choice == '6':
                self._voice_chat()
            elif choice == '7':
                self._lock_room()
            elif choice == '8':
                self._leave_room()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)
    finally:
        if self._current_room_name() is not None:
            self._leave_room(auto=True)
clear_screen()

Clear terminal screen.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1199
1200
1201
def clear_screen(self):
    """Clear terminal screen."""
    os.system('cls' if os.name == 'nt' else 'clear')
p2p_menu()

P2P configuration menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
    def p2p_menu(self):
        """P2P configuration menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ 🔧 P2P CONFIGURATION ───────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Start Relay Server')}  - Become a relay for P2P connections         {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Connect as Peer')}     - Connect to relay and other peers           {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Expose Local Service')} - Make local service accessible via P2P     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('Stop Instance')}       - Stop a running P2P instance                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}                - Return to main menu                        {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._start_relay()
            elif choice == '2':
                self._connect_peer()
            elif choice == '3':
                self._expose_service()
            elif choice == '4':
                self._stop_instance()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)
print_header()

Print main header.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1203
1204
1205
1206
1207
1208
1209
1210
    def print_header(self):
        """Print main header."""
        print(f"""
{Style.CYAN('╔══════════════════════════════════════════════════════════════════════╗')}
{Style.CYAN('║')} {Style.Bold(Style.WHITE('🌐 ToolBox P2P Manager'))} {Style.CYAN('v2.0')} {Style.GREY('- Interactive Mode')} {self._current_room_name() or '':<21} {Style.CYAN('║')}
{Style.CYAN('║')} {Style.GREY('E2E Encrypted Chat • File Transfer • Voice Chat • P2P Tunnels')} {' ' * 6} {Style.CYAN('║')}
{Style.CYAN('╚══════════════════════════════════════════════════════════════════════╝')}
""")
print_menu()

Print main menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
    def print_menu(self):
        """Print main menu."""
        print(f"""
{Style.Bold(Style.WHITE('┌─ 🎯 MAIN MENU ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('💬 Chat Mode')}          - Start interactive E2E encrypted chat     {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('🔧 P2P Configuration')}  - Configure P2P connections                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('📊 Status & Monitoring')} - View connections and rooms              {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('4.')} {Style.WHITE('⚙️  Settings')}           - Manage configuration                    {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('🚪 Exit')}               - Quit application                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")
run()

Main application loop.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
def run(self):
    """Main application loop."""
    while self.running:
        self.clear_screen()
        self.print_header()
        self.print_menu()

        choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

        if choice == '0':
            print(f"\n{Style.YELLOW('👋 Goodbye!')}")
            self.running = False
        elif choice == '1':
            self.chat_menu()
        elif choice == '2':
            self.p2p_menu()
        elif choice == '3':
            self.status_menu()
        elif choice == '4':
            self.settings_menu()
        else:
            print(f"{Style.RED('Invalid option')}")
            time.sleep(1)
settings_menu()

Settings menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
    def settings_menu(self):
        """Settings menu."""
        while True:
            self.clear_screen()
            self.print_header()

            print(f"""
{Style.Bold(Style.WHITE('┌─ ⚙️  SETTINGS ───────────────────────────────────────────────────────┐'))}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('1.')} {Style.WHITE('Change Username')}    - Set display name for chat                   {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('2.')} {Style.WHITE('Build P2P Binary')}   - Compile Rust P2P application                {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('3.')} {Style.WHITE('Clean Up')}           - Remove old instances and data               {Style.WHITE('│')}
{Style.WHITE('│')}  {Style.CYAN('0.')} {Style.WHITE('Back')}               - Return to main menu                         {Style.WHITE('│')}
{Style.WHITE('│')}                                                                      {Style.WHITE('│')}
{Style.Bold(Style.WHITE('└──────────────────────────────────────────────────────────────────────┘'))}
""")

            choice = input(f"\n{Style.CYAN('❯')} {Style.WHITE('Select option:')} ").strip()

            if choice == '0':
                break
            elif choice == '1':
                self._change_username()
            elif choice == '2':
                self._build_binary()
            elif choice == '3':
                self._cleanup()
            else:
                print(f"{Style.RED('Invalid option')}")
                time.sleep(1)
status_menu(do_clear=True)

Status and monitoring menu.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
def status_menu(self, do_clear=True):
    """Status and monitoring menu."""
    self.clear_screen() if do_clear else None
    self.print_header() if do_clear else None

    print(f"\n{Style.Bold(Style.CYAN('📊 System Status'))}")
    print(Style.GREY('═' * 90))

    # P2P Instances
    print(f"\n{Style.Bold(Style.WHITE('P2P Instances:'))}")
    if not self.instances:
        print(Style.YELLOW("  No instances configured"))
    else:
        print(
            f"\n{Style.Underline('NAME'):<20} {Style.Underline('MODE'):<12} {Style.Underline('STATUS'):<12} {Style.Underline('PID'):<10} {Style.Underline('CHAT ROOM')}")
        print(Style.GREY('─' * 90))

        for name, inst in self.instances.items():
            state = inst.read_state()
            mode = state.get('mode', 'Unknown')
            pid = state.get('pid', 'N/A')
            chat_room = state.get('chat_room', '-')
            status = Style.GREEN('✅ Running') if inst.is_running() else Style.RED('❌ Stopped')

            print(
                f"{Style.YELLOW(name):<20} {mode:<12} {status:<12} {str(pid):<10} {Style.CYAN(str(chat_room)[:20])}")

    # Chat Rooms
    print(f"\n{Style.Bold(Style.WHITE('Chat Rooms:'))}")
    result = self.chat_manager.list_rooms()

    if result.is_ok():
        rooms = result.get()
        if not rooms:
            print(Style.YELLOW("  No chat rooms"))
        else:
            print(
                f"\n{Style.Underline('NAME'):<20} {Style.Underline('PARTICIPANTS'):<15} {Style.Underline('STATUS'):<15} {Style.Underline('FEATURES')}")
            print(Style.GREY('─' * 70))

            for room in rooms:
                name = Style.YELLOW(room['name'][:18])
                participants = f"{room['participants_count']}/{room['max_participants']}"

                status_parts = []
                if room['is_locked']:
                    status_parts.append('🔒')
                if room['is_private']:
                    status_parts.append('🔐')
                if room['is_member']:
                    status_parts.append('✓')
                status = ' '.join(status_parts) if status_parts else '🔓'

                features = []
                if room['voice_enabled']:
                    features.append('🎤 Voice')
                if room['file_transfer_enabled']:
                    features.append('📁 Files')
                features_str = ', '.join(features)

                print(f"{name:<20} {participants:<15} {status:<15} {features_str}")

    input(f"\n{Style.GREY('Press Enter to continue...')}") if do_clear else None
P2PChatManager

Manages E2E encrypted chat rooms with file and voice support.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
class P2PChatManager:
    """Manages E2E encrypted chat rooms with file and voice support."""

    def __init__(self, app: App, voice_server_info: Dict[str, Tuple[str, int]]):
        self.app = app
        self.rooms: Dict[str, ChatRoom] = {}
        self.current_room: Optional[str] = None
        self.username = app.get_username() or "anonymous"
        CHAT_ROOMS_DIR.mkdir(parents=True, exist_ok=True)
        self._load_rooms()
        self.file_managers: Dict[str, FileTransferManager] = {}
        self.voice_manager: Optional[VoiceChatManager] = None
        self.voice_server_info = voice_server_info

    def create_room(self, name: str, password: str, max_participants: int = 10,
                    voice_enabled: bool = False, private: bool = False) -> Result:
        """Create a new chat room."""
        room_id = hashlib.sha256(f"{name}_{self.username}_{time.time()}".encode()).hexdigest()[:12]
        encryption_key = CryptoManager.generate_room_key(room_id, password)

        room = ChatRoom(
            room_id=room_id,
            name=name,
            owner=self.username,
            participants={self.username},
            is_locked=False,
            is_private=private,
            created_at=datetime.now(),
            encryption_key=encryption_key.decode(),
            max_participants=max_participants,
            voice_enabled=voice_enabled,
            file_transfer_enabled=True
        )

        self.rooms[room_id] = room
        self._save_room(room)
        # Start voice server if voice enabled
        if voice_enabled:
            try:
                voice_mgr = VoiceChatManager(room_id, encryption_key, self.username)
                port = voice_mgr.start_voice_server()
                self.voice_server_info[room_id] = ('127.0.0.1', port)
                print(f"   {Style.GREEN('Voice server started on port:')} {port}")
            except Exception as e:
                print(f"   {Style.YELLOW(f'Warning: Could not start voice server: {e}')}")
        # Send system message
        self._send_system_message(room_id, f"Room '{name}' created by {self.username}")

        return Result.ok(data={
            'room_id': room_id,
            'name': name,
            'message': f'Room "{name}" created successfully'
        })

    def join_room(self, room_id: str, password: str) -> Result:
        """Join an existing chat room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]

        if room.is_locked:
            return Result.default_user_error("Room is locked")

        if len(room.participants) >= room.max_participants:
            return Result.default_user_error("Room is full")

        # Verify password
        try:
            key = CryptoManager.generate_room_key(room_id, password)
            if key.decode() != room.encryption_key:
                return Result.default_user_error("Invalid password")
        except Exception:
            return Result.default_user_error("Invalid password")

        room.participants.add(self.username)
        self.current_room = room_id
        self._save_room(room)

        # Initialize file manager
        self.file_managers[room_id] = FileTransferManager(room_id, key)

        # Initialize voice manager if enabled
        # Get voice server info if available
        if room.voice_enabled:
            # Ask for voice server details if not already known
            if room_id not in self.voice_server_info:
                print(f"\n{Style.CYAN('Voice chat is enabled. Enter server details:')}")
                voice_host = input(
                    f"  {Style.WHITE('Voice server host (default: 127.0.0.1):')} ").strip() or "127.0.0.1"
                voice_port = input(f"  {Style.WHITE('Voice server port:')} ").strip()

                if voice_port and voice_port.isdigit():
                    self.voice_server_info[room_id] = (voice_host, int(voice_port))

        # Send system message
        self._send_system_message(room_id, f"{self.username} joined the room")

        return Result.ok(data={
            'room_id': room_id,
            'name': room.name,
            'participants': list(room.participants),
            'voice_enabled': room.voice_enabled,
            'file_transfer_enabled': room.file_transfer_enabled
        })

    def leave_room(self, room_id: str) -> Result:
        """Leave a chat room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        # Send system message before leaving
        self._send_system_message(room_id, f"{self.username} left the room")

        room.participants.remove(self.username)

        # If owner leaves, transfer ownership or delete room
        if room.owner == self.username:
            if len(room.participants) > 0:
                room.owner = list(room.participants)[0]
                self._send_system_message(room_id, f"Room ownership transferred to {room.owner}")
            else:
                # Delete empty room
                self._delete_room(room_id)
                return Result.ok(data="Room deleted (no participants)")

        self._save_room(room)

        if self.current_room == room_id:
            self.current_room = None

        # Cleanup managers
        if room_id in self.file_managers:
            del self.file_managers[room_id]
        if self.voice_manager:
            self.voice_manager.cleanup()
            self.voice_manager = None

        return Result.ok(data="Left room successfully")

    def lock_room(self, room_id: str) -> Result:
        """Lock a room to prevent new participants."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]

        if room.owner != self.username:
            return Result.default_user_error("Only room owner can lock the room")

        room.is_locked = True
        room.is_private = True
        self._save_room(room)

        self._send_system_message(room_id, f"Room locked by {self.username}")

        return Result.ok(data=f'Room "{room.name}" is now locked and private')

    def send_message(self, room_id: str, content: str, password: str) -> Result:
        """Send encrypted text message to room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        try:
            key = CryptoManager.generate_room_key(room_id, password)
            encrypted_content = CryptoManager.encrypt_message(content, key)

            message = ChatMessage(
                sender=self.username,
                content=encrypted_content,
                timestamp=datetime.now(),
                room_id=room_id,
                message_type=MessageType.TEXT,
                encrypted=True
            )

            self._save_message(message)
            return Result.ok(data="Message sent")

        except Exception as e:
            return Result.default_internal_error(f"Failed to send message: {e}")

    def send_file(self, room_id: str, file_path: Path, password: str) -> Result:
        """Send encrypted file to room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if not room.file_transfer_enabled:
            return Result.default_user_error("File transfer disabled in this room")

        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        try:
            # Prepare file for transfer
            file_manager = self.file_managers.get(room_id)
            if not file_manager:
                key = CryptoManager.generate_room_key(room_id, password)
                file_manager = FileTransferManager(room_id, key)
                self.file_managers[room_id] = file_manager

            transfer_id, file_size = file_manager.prepare_file(file_path)

            # Create file message
            key = CryptoManager.generate_room_key(room_id, password)
            encrypted_content = CryptoManager.encrypt_message(transfer_id, key)

            message = ChatMessage(
                sender=self.username,
                content=encrypted_content,
                timestamp=datetime.now(),
                room_id=room_id,
                message_type=MessageType.FILE,
                encrypted=True,
                file_name=file_path.name,
                file_size=file_size
            )

            self._save_message(message)

            return Result.ok(data={
                'transfer_id': transfer_id,
                'file_name': file_path.name,
                'file_size': file_size
            })

        except Exception as e:
            return Result.default_internal_error(f"Failed to send file: {e}")

    def receive_file(self, room_id: str, transfer_id: str, file_name: str) -> Result:
        """Receive and decrypt file from room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        try:
            file_manager = self.file_managers.get(room_id)
            if not file_manager:
                return Result.default_user_error("File manager not initialized")

            output_path = file_manager.receive_file(transfer_id, file_name)

            return Result.ok(data={
                'file_path': str(output_path),
                'file_name': file_name
            })

        except Exception as e:
            return Result.default_internal_error(f"Failed to receive file: {e}")

    def get_messages(self, room_id: str, password: str, limit: int = 50) -> Result:
        """Get decrypted messages from room."""
        if room_id not in self.rooms:
            return Result.default_user_error("Room not found")

        room = self.rooms[room_id]
        if self.username not in room.participants:
            return Result.default_user_error("You are not in this room")

        try:
            key = CryptoManager.generate_room_key(room_id, password)
            messages = self._load_messages(room_id, limit)

            decrypted_messages = []
            for msg in messages:
                if msg.encrypted and msg.message_type != MessageType.SYSTEM:
                    try:
                        decrypted_content = CryptoManager.decrypt_message(msg.content, key)
                        decrypted_messages.append({
                            'sender': msg.sender,
                            'content': decrypted_content,
                            'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                            'message_type': msg.message_type.value,
                            'is_own': msg.sender == self.username,
                            'file_name': msg.file_name,
                            'file_size': msg.file_size
                        })
                    except Exception:
                        continue
                else:
                    decrypted_messages.append({
                        'sender': msg.sender,
                        'content': msg.content,
                        'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                        'message_type': msg.message_type.value,
                        'is_own': False
                    })

            return Result.ok(data=decrypted_messages)

        except Exception as e:
            return Result.default_internal_error(f"Failed to get messages: {e}")

    def list_rooms(self, show_all: bool = False) -> Result:
        """List available rooms for user."""
        user_rooms = []
        for room in self.rooms.values():
            # Show only user's rooms unless show_all is True
            if show_all or self.username in room.participants:
                # Don't show private/locked rooms to non-participants
                if room.is_private and self.username not in room.participants:
                    continue

                user_rooms.append({
                    'room_id': room.room_id,
                    'name': room.name,
                    'owner': room.owner,
                    'participants_count': len(room.participants),
                    'max_participants': room.max_participants,
                    'is_locked': room.is_locked,
                    'is_private': room.is_private,
                    'voice_enabled': room.voice_enabled,
                    'file_transfer_enabled': room.file_transfer_enabled,
                    'created_at': room.created_at.strftime('%Y-%m-%d %H:%M'),
                    'is_member': self.username in room.participants
                })

        return Result.ok(data=user_rooms)

    def _send_system_message(self, room_id: str, content: str):
        """Send a system message (not encrypted)."""
        message = ChatMessage(
            sender="SYSTEM",
            content=content,
            timestamp=datetime.now(),
            room_id=room_id,
            message_type=MessageType.SYSTEM,
            encrypted=False
        )
        self._save_message(message)

    def _save_room(self, room: ChatRoom):
        """Save room to storage."""
        room_file = CHAT_ROOMS_DIR / f"room_{room.room_id}.json"

        with open(room_file, 'w') as f:
            json.dump(room.to_dict(), f, indent=2)

    def _load_rooms(self):
        """Load rooms from storage."""
        if not CHAT_ROOMS_DIR.exists():
            return

        for room_file in CHAT_ROOMS_DIR.glob("room_*.json"):
            try:
                with open(room_file) as f:
                    room_data = json.load(f)
                    room = ChatRoom.from_dict(room_data)
                    self.rooms[room.room_id] = room
            except Exception as e:
                print(f"Warning: Failed to load room {room_file}: {e}")

    def _delete_room(self, room_id: str):
        """Delete a room and its messages."""
        if room_id in self.rooms:
            del self.rooms[room_id]

        # Delete room file
        room_file = CHAT_ROOMS_DIR / f"room_{room_id}.json"
        if room_file.exists():
            room_file.unlink()

        # Delete messages file
        messages_file = CHAT_ROOMS_DIR / f"messages_{room_id}.jsonl"
        if messages_file.exists():
            messages_file.unlink()

    def _save_message(self, message: ChatMessage):
        """Save message to storage."""
        messages_file = CHAT_ROOMS_DIR / f"messages_{message.room_id}.jsonl"
        with open(messages_file, 'a') as f:
            f.write(json.dumps(message.to_dict()) + '\n')

    def _load_messages(self, room_id: str, limit: int = 50) -> List[ChatMessage]:
        """Load messages from storage."""
        messages_file = CHAT_ROOMS_DIR / f"messages_{room_id}.jsonl"
        if not messages_file.exists():
            return []

        messages = []
        with open(messages_file) as f:
            lines = f.readlines()
            for line in lines[-limit:]:
                try:
                    message_data = json.loads(line.strip())
                    messages.append(ChatMessage.from_dict(message_data))
                except Exception:
                    continue

        return messages
create_room(name, password, max_participants=10, voice_enabled=False, private=False)

Create a new chat room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
def create_room(self, name: str, password: str, max_participants: int = 10,
                voice_enabled: bool = False, private: bool = False) -> Result:
    """Create a new chat room."""
    room_id = hashlib.sha256(f"{name}_{self.username}_{time.time()}".encode()).hexdigest()[:12]
    encryption_key = CryptoManager.generate_room_key(room_id, password)

    room = ChatRoom(
        room_id=room_id,
        name=name,
        owner=self.username,
        participants={self.username},
        is_locked=False,
        is_private=private,
        created_at=datetime.now(),
        encryption_key=encryption_key.decode(),
        max_participants=max_participants,
        voice_enabled=voice_enabled,
        file_transfer_enabled=True
    )

    self.rooms[room_id] = room
    self._save_room(room)
    # Start voice server if voice enabled
    if voice_enabled:
        try:
            voice_mgr = VoiceChatManager(room_id, encryption_key, self.username)
            port = voice_mgr.start_voice_server()
            self.voice_server_info[room_id] = ('127.0.0.1', port)
            print(f"   {Style.GREEN('Voice server started on port:')} {port}")
        except Exception as e:
            print(f"   {Style.YELLOW(f'Warning: Could not start voice server: {e}')}")
    # Send system message
    self._send_system_message(room_id, f"Room '{name}' created by {self.username}")

    return Result.ok(data={
        'room_id': room_id,
        'name': name,
        'message': f'Room "{name}" created successfully'
    })
get_messages(room_id, password, limit=50)

Get decrypted messages from room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
def get_messages(self, room_id: str, password: str, limit: int = 50) -> Result:
    """Get decrypted messages from room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    try:
        key = CryptoManager.generate_room_key(room_id, password)
        messages = self._load_messages(room_id, limit)

        decrypted_messages = []
        for msg in messages:
            if msg.encrypted and msg.message_type != MessageType.SYSTEM:
                try:
                    decrypted_content = CryptoManager.decrypt_message(msg.content, key)
                    decrypted_messages.append({
                        'sender': msg.sender,
                        'content': decrypted_content,
                        'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                        'message_type': msg.message_type.value,
                        'is_own': msg.sender == self.username,
                        'file_name': msg.file_name,
                        'file_size': msg.file_size
                    })
                except Exception:
                    continue
            else:
                decrypted_messages.append({
                    'sender': msg.sender,
                    'content': msg.content,
                    'timestamp': msg.timestamp.strftime('%H:%M:%S'),
                    'message_type': msg.message_type.value,
                    'is_own': False
                })

        return Result.ok(data=decrypted_messages)

    except Exception as e:
        return Result.default_internal_error(f"Failed to get messages: {e}")
join_room(room_id, password)

Join an existing chat room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
def join_room(self, room_id: str, password: str) -> Result:
    """Join an existing chat room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]

    if room.is_locked:
        return Result.default_user_error("Room is locked")

    if len(room.participants) >= room.max_participants:
        return Result.default_user_error("Room is full")

    # Verify password
    try:
        key = CryptoManager.generate_room_key(room_id, password)
        if key.decode() != room.encryption_key:
            return Result.default_user_error("Invalid password")
    except Exception:
        return Result.default_user_error("Invalid password")

    room.participants.add(self.username)
    self.current_room = room_id
    self._save_room(room)

    # Initialize file manager
    self.file_managers[room_id] = FileTransferManager(room_id, key)

    # Initialize voice manager if enabled
    # Get voice server info if available
    if room.voice_enabled:
        # Ask for voice server details if not already known
        if room_id not in self.voice_server_info:
            print(f"\n{Style.CYAN('Voice chat is enabled. Enter server details:')}")
            voice_host = input(
                f"  {Style.WHITE('Voice server host (default: 127.0.0.1):')} ").strip() or "127.0.0.1"
            voice_port = input(f"  {Style.WHITE('Voice server port:')} ").strip()

            if voice_port and voice_port.isdigit():
                self.voice_server_info[room_id] = (voice_host, int(voice_port))

    # Send system message
    self._send_system_message(room_id, f"{self.username} joined the room")

    return Result.ok(data={
        'room_id': room_id,
        'name': room.name,
        'participants': list(room.participants),
        'voice_enabled': room.voice_enabled,
        'file_transfer_enabled': room.file_transfer_enabled
    })
leave_room(room_id)

Leave a chat room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
def leave_room(self, room_id: str) -> Result:
    """Leave a chat room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    # Send system message before leaving
    self._send_system_message(room_id, f"{self.username} left the room")

    room.participants.remove(self.username)

    # If owner leaves, transfer ownership or delete room
    if room.owner == self.username:
        if len(room.participants) > 0:
            room.owner = list(room.participants)[0]
            self._send_system_message(room_id, f"Room ownership transferred to {room.owner}")
        else:
            # Delete empty room
            self._delete_room(room_id)
            return Result.ok(data="Room deleted (no participants)")

    self._save_room(room)

    if self.current_room == room_id:
        self.current_room = None

    # Cleanup managers
    if room_id in self.file_managers:
        del self.file_managers[room_id]
    if self.voice_manager:
        self.voice_manager.cleanup()
        self.voice_manager = None

    return Result.ok(data="Left room successfully")
list_rooms(show_all=False)

List available rooms for user.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
def list_rooms(self, show_all: bool = False) -> Result:
    """List available rooms for user."""
    user_rooms = []
    for room in self.rooms.values():
        # Show only user's rooms unless show_all is True
        if show_all or self.username in room.participants:
            # Don't show private/locked rooms to non-participants
            if room.is_private and self.username not in room.participants:
                continue

            user_rooms.append({
                'room_id': room.room_id,
                'name': room.name,
                'owner': room.owner,
                'participants_count': len(room.participants),
                'max_participants': room.max_participants,
                'is_locked': room.is_locked,
                'is_private': room.is_private,
                'voice_enabled': room.voice_enabled,
                'file_transfer_enabled': room.file_transfer_enabled,
                'created_at': room.created_at.strftime('%Y-%m-%d %H:%M'),
                'is_member': self.username in room.participants
            })

    return Result.ok(data=user_rooms)
lock_room(room_id)

Lock a room to prevent new participants.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
def lock_room(self, room_id: str) -> Result:
    """Lock a room to prevent new participants."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]

    if room.owner != self.username:
        return Result.default_user_error("Only room owner can lock the room")

    room.is_locked = True
    room.is_private = True
    self._save_room(room)

    self._send_system_message(room_id, f"Room locked by {self.username}")

    return Result.ok(data=f'Room "{room.name}" is now locked and private')
receive_file(room_id, transfer_id, file_name)

Receive and decrypt file from room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
def receive_file(self, room_id: str, transfer_id: str, file_name: str) -> Result:
    """Receive and decrypt file from room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    try:
        file_manager = self.file_managers.get(room_id)
        if not file_manager:
            return Result.default_user_error("File manager not initialized")

        output_path = file_manager.receive_file(transfer_id, file_name)

        return Result.ok(data={
            'file_path': str(output_path),
            'file_name': file_name
        })

    except Exception as e:
        return Result.default_internal_error(f"Failed to receive file: {e}")
send_file(room_id, file_path, password)

Send encrypted file to room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
def send_file(self, room_id: str, file_path: Path, password: str) -> Result:
    """Send encrypted file to room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if not room.file_transfer_enabled:
        return Result.default_user_error("File transfer disabled in this room")

    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    try:
        # Prepare file for transfer
        file_manager = self.file_managers.get(room_id)
        if not file_manager:
            key = CryptoManager.generate_room_key(room_id, password)
            file_manager = FileTransferManager(room_id, key)
            self.file_managers[room_id] = file_manager

        transfer_id, file_size = file_manager.prepare_file(file_path)

        # Create file message
        key = CryptoManager.generate_room_key(room_id, password)
        encrypted_content = CryptoManager.encrypt_message(transfer_id, key)

        message = ChatMessage(
            sender=self.username,
            content=encrypted_content,
            timestamp=datetime.now(),
            room_id=room_id,
            message_type=MessageType.FILE,
            encrypted=True,
            file_name=file_path.name,
            file_size=file_size
        )

        self._save_message(message)

        return Result.ok(data={
            'transfer_id': transfer_id,
            'file_name': file_path.name,
            'file_size': file_size
        })

    except Exception as e:
        return Result.default_internal_error(f"Failed to send file: {e}")
send_message(room_id, content, password)

Send encrypted text message to room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
def send_message(self, room_id: str, content: str, password: str) -> Result:
    """Send encrypted text message to room."""
    if room_id not in self.rooms:
        return Result.default_user_error("Room not found")

    room = self.rooms[room_id]
    if self.username not in room.participants:
        return Result.default_user_error("You are not in this room")

    try:
        key = CryptoManager.generate_room_key(room_id, password)
        encrypted_content = CryptoManager.encrypt_message(content, key)

        message = ChatMessage(
            sender=self.username,
            content=encrypted_content,
            timestamp=datetime.now(),
            room_id=room_id,
            message_type=MessageType.TEXT,
            encrypted=True
        )

        self._save_message(message)
        return Result.ok(data="Message sent")

    except Exception as e:
        return Result.default_internal_error(f"Failed to send message: {e}")
P2PConnection dataclass

Represents a P2P connection configuration.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
133
134
135
136
137
138
139
140
141
@dataclass
class P2PConnection:
    """Represents a P2P connection configuration."""
    name: str
    mode: str  # relay, peer-provider, peer-consumer
    status: str  # active, stopped, error
    pid: Optional[int] = None
    config: dict = field(default_factory=dict)
    chat_room: Optional[str] = None
VoiceChatManager

Manages P2P voice chat with live streaming and speaker detection.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
class VoiceChatManager:
    """Manages P2P voice chat with live streaming and speaker detection."""

    def __init__(self, room_id: str, encryption_key: bytes, username: str):
        self.room_id = room_id
        self.encryption_key = encryption_key
        self.username = username
        self.is_recording = False
        self.is_playing = False
        self.current_speaker = None
        self.voice_server_port = None

        if not VOICE_ENABLED:
            return
            raise RuntimeError("pyaudio not installed. Install with: pip install pyaudio")

        self.audio = pyaudio.PyAudio()
        self.voice_dir = VOICE_CACHE_DIR / room_id
        self.voice_dir.mkdir(parents=True, exist_ok=True)

        # Voice activity detection
        self.voice_threshold = 500  # Audio level threshold
        self.speaking = False

        # Network
        self.server_socket = None
        self.clients = {}  # {addr: socket}
        self.running = False

    def calculate_rms(self, audio_data):
        """Calculate RMS (Root Mean Square) for voice activity detection."""
        import array
        count = len(audio_data) / 2
        format_str = "%dh" % count
        shorts = array.array('h', audio_data)
        sum_squares = sum((sample ** 2 for sample in shorts))
        rms = (sum_squares / count) ** 0.5
        return rms

    def start_voice_server(self, port: int = 0):
        """Start voice relay server for this room."""
        self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.server_socket.bind(('0.0.0.0', port))
        self.server_socket.listen(5)
        self.server_socket.settimeout(0.5)

        self.voice_server_port = self.server_socket.getsockname()[1]
        self.running = True

        # Start accepting clients
        accept_thread = threading.Thread(target=self._accept_clients, daemon=True)
        accept_thread.start()

        return self.voice_server_port

    def _accept_clients(self):
        """Accept incoming voice client connections."""
        while self.running:
            try:
                client_sock, addr = self.server_socket.accept()
                client_sock.settimeout(1.0)
                self.clients[addr] = client_sock
                print(f"\r{Style.GREEN('🎤 New voice participant connected')}{' ' * 20}")

                # Start receiving thread for this client
                recv_thread = threading.Thread(
                    target=self._receive_from_client,
                    args=(client_sock, addr),
                    daemon=True
                )
                recv_thread.start()

            except socket.timeout:
                continue
            except Exception as e:
                if self.running:
                    print(f"\r{Style.RED(f'Voice server error: {e}')}{' ' * 20}")

    def _receive_from_client(self, client_sock, addr):
        """Receive audio from client and broadcast to others."""
        try:
            while self.running:
                try:
                    # Receive packet size
                    size_data = client_sock.recv(4)
                    if not size_data or len(size_data) < 4:
                        break

                    packet_size = int.from_bytes(size_data, 'big')

                    # Sanity check
                    if packet_size > 1024 * 1024:  # 1MB max
                        break

                    # Receive full packet
                    packet = b''
                    while len(packet) < packet_size:
                        remaining = packet_size - len(packet)
                        chunk = client_sock.recv(min(remaining, 4096))
                        if not chunk:
                            break
                        packet += chunk

                    if len(packet) != packet_size:
                        break

                    # Parse packet header to update speaker
                    if len(packet) >= 3:
                        username_len = int.from_bytes(packet[:2], 'big')
                        if len(packet) >= 2 + username_len + 1:
                            username = packet[2:2 + username_len].decode('utf-8')
                            is_speaking = packet[2 + username_len] == 1

                            # Update current speaker
                            if is_speaking:
                                self.current_speaker = username
                            elif self.current_speaker == username:
                                self.current_speaker = None

                    # Broadcast to all other clients
                    self._broadcast_audio(packet, addr)

                except socket.timeout:
                    continue
                except Exception:
                    break

        except Exception:
            pass
        finally:
            if addr in self.clients:
                del self.clients[addr]
                print(f"\r{Style.YELLOW('Voice participant disconnected')}{' ' * 20}")
            try:
                client_sock.close()
            except:
                pass

    def _broadcast_audio(self, packet, exclude_addr):
        """Broadcast audio packet to all clients except sender."""
        dead_clients = []
        for addr, client_sock in self.clients.items():
            if addr == exclude_addr:
                continue
            try:
                # Send packet size then packet
                size_bytes = len(packet).to_bytes(4, 'big')
                client_sock.sendall(size_bytes + packet)
            except:
                dead_clients.append(addr)

        # Remove dead clients
        for addr in dead_clients:
            if addr in self.clients:
                del self.clients[addr]

    def connect_to_voice_server(self, host: str, port: int):
        """Connect to voice relay server."""
        self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.client_socket.connect((host, port))
        self.client_socket.settimeout(1.0)
        self.running = True

        # Start playback thread
        playback_thread = threading.Thread(target=self._playback_loop, daemon=True)
        playback_thread.start()

    def _playback_loop(self):
        """Receive and play audio from server."""
        stream = self.audio.open(
            format=VOICE_FORMAT,
            channels=VOICE_CHANNELS,
            rate=VOICE_RATE,
            output=True,
            frames_per_buffer=VOICE_CHUNK
        )

        print(f"{Style.GREEN('🔊 Playback active - Listening...')}")

        try:
            while self.running:
                try:
                    # Receive packet size
                    size_data = self.client_socket.recv(4)
                    if not size_data or len(size_data) < 4:
                        break

                    packet_size = int.from_bytes(size_data, 'big')

                    # Sanity check
                    if packet_size > 1024 * 1024:  # 1MB max
                        print(f"\r{Style.RED('Invalid packet size')}{' ' * 20}")
                        break

                    # Receive full packet
                    packet = b''
                    while len(packet) < packet_size:
                        remaining = packet_size - len(packet)
                        chunk = self.client_socket.recv(min(remaining, 4096))
                        if not chunk:
                            break
                        packet += chunk

                    if len(packet) != packet_size:
                        break

                    # Parse packet
                    if len(packet) < 3:
                        continue

                    username_len = int.from_bytes(packet[:2], 'big')
                    if len(packet) < 2 + username_len + 1:
                        continue

                    username = packet[2:2 + username_len].decode('utf-8')
                    is_speaking = packet[2 + username_len] == 1
                    audio_data = packet[2 + username_len + 1:]

                    # Update speaker
                    if is_speaking:
                        self.current_speaker = username
                    elif self.current_speaker == username:
                        self.current_speaker = None

                    # Decrypt and play if there's audio data
                    if len(audio_data) > 0:
                        try:
                            decrypted_audio = CryptoManager.decrypt_bytes(
                                audio_data,
                                self.encryption_key
                            )
                            stream.write(decrypted_audio)
                        except Exception as e:
                            # Decryption failed, skip this packet
                            pass

                except socket.timeout:
                    continue
                except Exception as e:
                    if self.running:
                        print(f"\r{Style.RED(f'Playback error: {e}')}{' ' * 20}")
                    break

        finally:
            stream.stop_stream()
            stream.close()
            print(f"\n{Style.YELLOW('🔇 Playback stopped')}")

    def start_recording_stream(self):
        """Start streaming microphone input to server."""
        self.is_recording = True

        stream = self.audio.open(
            format=VOICE_FORMAT,
            channels=VOICE_CHANNELS,
            rate=VOICE_RATE,
            input=True,
            frames_per_buffer=VOICE_CHUNK
        )

        print(f"{Style.GREEN('🎤 Microphone active - Start speaking!')}")

        try:
            silence_counter = 0
            while self.is_recording:
                try:
                    # Read audio
                    audio_data = stream.read(VOICE_CHUNK, exception_on_overflow=False)

                    # Voice activity detection
                    rms = self.calculate_rms(audio_data)
                    is_speaking = rms > self.voice_threshold

                    if is_speaking:
                        self.speaking = True
                        silence_counter = 0

                        # Encrypt audio directly as bytes
                        encrypted_bytes = CryptoManager.encrypt_bytes(
                            audio_data,
                            self.encryption_key
                        )

                        # Build packet: [username_len(2)][username][speaker_flag(1)][audio_data]
                        username_bytes = self.username.encode('utf-8')
                        username_len = len(username_bytes).to_bytes(2, 'big')
                        speaker_flag = b'\x01'

                        packet = username_len + username_bytes + speaker_flag + encrypted_bytes

                        # Send to server
                        try:
                            size_bytes = len(packet).to_bytes(4, 'big')
                            self.client_socket.sendall(size_bytes + packet)
                        except Exception as e:
                            print(f"\r{Style.RED(f'Send error: {e}')}{' ' * 20}")
                            break
                    else:
                        silence_counter += 1

                        # Send stop-speaking packet after 3 consecutive silent chunks
                        if self.speaking and silence_counter > 3:
                            username_bytes = self.username.encode('utf-8')
                            username_len = len(username_bytes).to_bytes(2, 'big')
                            packet = username_len + username_bytes + b'\x00'

                            try:
                                size_bytes = len(packet).to_bytes(4, 'big')
                                self.client_socket.sendall(size_bytes + packet)
                            except:
                                pass

                            self.speaking = False

                except Exception as e:
                    if self.is_recording:
                        print(f"\r{Style.RED(f'Recording error: {e}')}{' ' * 20}")
                    break

        finally:
            stream.stop_stream()
            stream.close()
            print(f"\n{Style.YELLOW('🔇 Microphone stopped')}")

    def stop_recording(self):
        """Stop recording stream."""
        self.is_recording = False

    def get_current_speaker(self):
        """Get username of current speaker."""
        return self.current_speaker

    def cleanup(self):
        """Cleanup voice resources."""
        self.running = False
        self.is_recording = False

        if hasattr(self, 'client_socket'):
            try:
                self.client_socket.close()
            except:
                pass

        if self.server_socket:
            try:
                self.server_socket.close()
            except:
                pass

        # Close all client connections
        for client_sock in self.clients.values():
            try:
                client_sock.close()
            except:
                pass

        self.clients.clear()

        try:
            self.audio.terminate()
        except:
            pass
calculate_rms(audio_data)

Calculate RMS (Root Mean Square) for voice activity detection.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
283
284
285
286
287
288
289
290
291
def calculate_rms(self, audio_data):
    """Calculate RMS (Root Mean Square) for voice activity detection."""
    import array
    count = len(audio_data) / 2
    format_str = "%dh" % count
    shorts = array.array('h', audio_data)
    sum_squares = sum((sample ** 2 for sample in shorts))
    rms = (sum_squares / count) ** 0.5
    return rms
cleanup()

Cleanup voice resources.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
def cleanup(self):
    """Cleanup voice resources."""
    self.running = False
    self.is_recording = False

    if hasattr(self, 'client_socket'):
        try:
            self.client_socket.close()
        except:
            pass

    if self.server_socket:
        try:
            self.server_socket.close()
        except:
            pass

    # Close all client connections
    for client_sock in self.clients.values():
        try:
            client_sock.close()
        except:
            pass

    self.clients.clear()

    try:
        self.audio.terminate()
    except:
        pass
connect_to_voice_server(host, port)

Connect to voice relay server.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
411
412
413
414
415
416
417
418
419
420
def connect_to_voice_server(self, host: str, port: int):
    """Connect to voice relay server."""
    self.client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.client_socket.connect((host, port))
    self.client_socket.settimeout(1.0)
    self.running = True

    # Start playback thread
    playback_thread = threading.Thread(target=self._playback_loop, daemon=True)
    playback_thread.start()
get_current_speaker()

Get username of current speaker.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
583
584
585
def get_current_speaker(self):
    """Get username of current speaker."""
    return self.current_speaker
start_recording_stream()

Start streaming microphone input to server.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
def start_recording_stream(self):
    """Start streaming microphone input to server."""
    self.is_recording = True

    stream = self.audio.open(
        format=VOICE_FORMAT,
        channels=VOICE_CHANNELS,
        rate=VOICE_RATE,
        input=True,
        frames_per_buffer=VOICE_CHUNK
    )

    print(f"{Style.GREEN('🎤 Microphone active - Start speaking!')}")

    try:
        silence_counter = 0
        while self.is_recording:
            try:
                # Read audio
                audio_data = stream.read(VOICE_CHUNK, exception_on_overflow=False)

                # Voice activity detection
                rms = self.calculate_rms(audio_data)
                is_speaking = rms > self.voice_threshold

                if is_speaking:
                    self.speaking = True
                    silence_counter = 0

                    # Encrypt audio directly as bytes
                    encrypted_bytes = CryptoManager.encrypt_bytes(
                        audio_data,
                        self.encryption_key
                    )

                    # Build packet: [username_len(2)][username][speaker_flag(1)][audio_data]
                    username_bytes = self.username.encode('utf-8')
                    username_len = len(username_bytes).to_bytes(2, 'big')
                    speaker_flag = b'\x01'

                    packet = username_len + username_bytes + speaker_flag + encrypted_bytes

                    # Send to server
                    try:
                        size_bytes = len(packet).to_bytes(4, 'big')
                        self.client_socket.sendall(size_bytes + packet)
                    except Exception as e:
                        print(f"\r{Style.RED(f'Send error: {e}')}{' ' * 20}")
                        break
                else:
                    silence_counter += 1

                    # Send stop-speaking packet after 3 consecutive silent chunks
                    if self.speaking and silence_counter > 3:
                        username_bytes = self.username.encode('utf-8')
                        username_len = len(username_bytes).to_bytes(2, 'big')
                        packet = username_len + username_bytes + b'\x00'

                        try:
                            size_bytes = len(packet).to_bytes(4, 'big')
                            self.client_socket.sendall(size_bytes + packet)
                        except:
                            pass

                        self.speaking = False

            except Exception as e:
                if self.is_recording:
                    print(f"\r{Style.RED(f'Recording error: {e}')}{' ' * 20}")
                break

    finally:
        stream.stop_stream()
        stream.close()
        print(f"\n{Style.YELLOW('🔇 Microphone stopped')}")
start_voice_server(port=0)

Start voice relay server for this room.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
def start_voice_server(self, port: int = 0):
    """Start voice relay server for this room."""
    self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    self.server_socket.bind(('0.0.0.0', port))
    self.server_socket.listen(5)
    self.server_socket.settimeout(0.5)

    self.voice_server_port = self.server_socket.getsockname()[1]
    self.running = True

    # Start accepting clients
    accept_thread = threading.Thread(target=self._accept_clients, daemon=True)
    accept_thread.start()

    return self.voice_server_port
stop_recording()

Stop recording stream.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
579
580
581
def stop_recording(self):
    """Stop recording stream."""
    self.is_recording = False
cli_tcm_runner()

Main CLI entry point.

Source code in toolboxv2/utils/clis/tcm_p2p_cli.py
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
def cli_tcm_runner():
    """Main CLI entry point."""
    parser = argparse.ArgumentParser(
        description=f"🚀 {Style.Bold('ToolBox P2P Manager')} - Advanced P2P with E2E Chat",
        formatter_class=argparse.RawTextHelpFormatter
    )

    parser.add_argument('--interactive', '-i', action='store_true',
                        help='Start interactive mode (default)')
    parser.add_argument("status", nargs='?', const=True,
                        help='Check status of all instances')
    args = parser.parse_args()

    # Always start in interactive mode
    cli = InteractiveP2PCLI()

    if args.status:
        cli.status_menu(do_clear=False)
        return

    try:
        cli.run()
    except KeyboardInterrupt:
        print(f"\n\n{Style.YELLOW('👋 Interrupted by user. Goodbye!')}")
    except Exception as e:
        print(f"\n{Style.RED2('❌ Fatal error:')} {e}")
        import traceback
        traceback.print_exc()
    finally:
        # Cleanup
        print(f"\n{Style.GREY('Cleaning up...')}")
user_dashboard
interactive_user_dashboard() async

Modern interactive user dashboard and mini CLI

Source code in toolboxv2/utils/clis/user_dashboard.py
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
async def interactive_user_dashboard():
    """Modern interactive user dashboard and mini CLI"""
    import asyncio
    from pathlib import Path

    # =================== UI Helper Functions ===================
    # Note: print_box_header, print_box_content, print_box_footer, print_status, print_separator
    # are now imported from cli_printing at the top of the file

    def get_key():
        """Get single keypress (cross-platform)"""
        if system() == "Windows":
            import msvcrt
            key = msvcrt.getch()
            if key == b'\xe0':  # Arrow key prefix
                key = msvcrt.getch()
                if key == b'H':
                    return 'up'
                elif key == b'P':
                    return 'down'
            elif key == b'\r':
                return 'enter'
            elif key in (b'q', b'Q', b'\x03'):
                return 'quit'
            elif key in (b'w', b'W'):
                return 'up'
            elif key in (b's', b'S'):
                return 'down'
            elif key in (b'/', b'?'):
                return 'search'
            elif key in (b'h', b'H'):
                return 'help'
            return key.decode('utf-8', errors='ignore')
        else:
            import tty
            import termios
            fd = sys.stdin.fileno()
            old_settings = termios.tcgetattr(fd)
            try:
                tty.setraw(sys.stdin.fileno())
                ch = sys.stdin.read(1)
                if ch == '\x1b':  # ESC sequence
                    next_chars = sys.stdin.read(2)
                    if next_chars == '[A':
                        return 'up'
                    elif next_chars == '[B':
                        return 'down'
                elif ch in ('\r', '\n'):
                    return 'enter'
                elif ch in ('q', 'Q', '\x03'):
                    return 'quit'
                elif ch in ('w', 'W'):
                    return 'up'
                elif ch in ('s', 'S'):
                    return 'down'
                elif ch in ('/', '?'):
                    return 'search'
                elif ch in ('h', 'H'):
                    return 'help'
                return ch
            finally:
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

    # =================== Dashboard Manager ===================

    class DashboardManager:
        """Manages the interactive dashboard"""

        def __init__(self, app):
            self.app = app
            self.current_view = "main_menu"
            self.selected_index = 0
            self.running = True
            self.history = []
            self.search_query = ""

            # Cache
            self.modules_cache = None
            self.current_module = None
            self.current_functions = []

        async def run(self):
            """Main run loop"""
            # Clear screen
            print('\033[2J\033[H')

            # Welcome
            print_box_header("ToolBoxV2 Interactive Dashboard", "🎯")
            print_box_content("Welcome to the ToolBoxV2 Command Center", "info")
            print_box_footer()

            await asyncio.sleep(1)

            while self.running:
                try:
                    if self.current_view == "main_menu":
                        await self.show_main_menu()
                    elif self.current_view == "modules":
                        await self.show_modules()
                    elif self.current_view == "module_detail":
                        await self.show_module_detail()
                    elif self.current_view == "function_execute":
                        await self.execute_function()
                    elif self.current_view == "function_runner":
                        await self.show_function_runner()
                    elif self.current_view == "workflow_runner":
                        await self.show_workflow_runner()
                    elif self.current_view == "status":
                        await self.show_status()
                    elif self.current_view == "services":
                        await self.show_services()
                    elif self.current_view == "quick_actions":
                        await self.show_quick_actions()
                    elif self.current_view == "search":
                        await self.show_search()
                    elif self.current_view == "settings":
                        await self.show_settings()
                except KeyboardInterrupt:
                    if await self.confirm_exit():
                        break
                    continue

        async def show_main_menu(self):
            """Show main menu"""
            menu_items = [
                ("📦", "Browse Modules", "modules"),
                ("⚡", "Quick Actions", "quick_actions"),
                ("🎯", "Function Runner", "function_runner"),
                ("⏩", "Workflow Runner", "workflow_runner"),
                ("🔧", "Manage Services", "services"),
                ("📊", "System Status", "status"),
                ("🔍", "Search", "search"),
                ("⚙️", "Settings", "settings"),
                ("❌", "Exit", "exit")
            ]

            while True:
                print('\033[2J\033[H')

                print_box_header("Main Menu", "🏠")
                print()

                # User info
                username = self.app.get_username() if hasattr(self.app, 'get_username') else "Guest"
                print(f"  👤 User: {username}")
                print(f"  📍 Instance: {self.app.id}")
                print(f"  🖥️  System: {system()}")
                print()
                print_separator()
                print()

                # Menu items
                for i, (icon, label, _) in enumerate(menu_items):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_box_footer()
                print_status("↑↓/w/s: Navigate | Enter: Select | h: Help | q: Quit", "info")

                key = get_key()

                if key == 'quit':
                    if await self.confirm_exit():
                        self.running = False
                        return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(menu_items) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action = menu_items[self.selected_index]

                    if action == "exit":
                        if await self.confirm_exit():
                            self.running = False
                            return
                    elif action == "function_runner":
                        self.history.append(self.current_view)
                        self.current_view = "function_runner"
                        self.selected_index = 0
                        return
                    elif action == "workflow_runner":
                        self.history.append(self.current_view)
                        self.current_view = "workflow_runner"
                        self.selected_index = 0
                        return
                    else:
                        self.history.append(self.current_view)
                        self.current_view = action
                        self.selected_index = 0
                        return
                elif key == 'search':
                    self.history.append(self.current_view)
                    self.current_view = "search"
                    return
                elif key == 'help':
                    await self.show_help()

        async def show_modules(self):
            """Show modules list"""
            if self.modules_cache is None:
                print_status("Loading modules...", "progress")
                self.modules_cache = list(self.app.functions.keys())
                self.modules_cache.sort()

            while True:
                print('\033[2J\033[H')

                print_box_header("Module Browser", "📦")
                print_box_content(f"Total modules: {len(self.modules_cache)}", "info")
                print_box_footer()

                if not self.modules_cache:
                    print_status("No modules loaded", "warning")
                    print_status("Use -l flag to load all modules", "info")
                    print()
                    print_status("Press any key to go back...", "info")
                    get_key()
                    self.go_back()
                    return

                # Calculate visible range
                visible_count = 15
                start_idx = max(0, self.selected_index - visible_count // 2)
                end_idx = min(len(self.modules_cache), start_idx + visible_count)

                if end_idx - start_idx < visible_count:
                    start_idx = max(0, end_idx - visible_count)

                # Show modules
                print()
                for i in range(start_idx, end_idx):
                    module_name = self.modules_cache[i]
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    # Get module version if available
                    try:
                        mod = self.app.get_mod(module_name)
                        version = getattr(mod, 'version', '?.?.?')
                    except:
                        version = '?.?.?'

                    if is_selected:
                        print(f"  {arrow} \033[1;96m📦 {module_name:<30} v{version}\033[0m")
                    else:
                        print(f"  {arrow} 📦 {module_name:<30} v{version}")

                if len(self.modules_cache) > visible_count:
                    print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.modules_cache)}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Open | /: Search | b/Esc: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(self.modules_cache) - 1, self.selected_index + 1)
                elif key == 'enter':
                    self.current_module = self.modules_cache[self.selected_index]
                    self.history.append(self.current_view)
                    self.current_view = "module_detail"
                    self.selected_index = 0
                    return
                elif key == 'search':
                    self.history.append(self.current_view)
                    self.current_view = "search"
                    return

        async def show_module_detail(self):
            """Show module detail with functions"""
            if not self.current_module:
                self.go_back()
                return

            # Load functions
            print_status(f"Loading functions from {self.current_module}...", "progress")

            module_data = self.app.functions.get(self.current_module, {})
            self.current_functions = []

            for func_name, func_data in module_data.items():
                if isinstance(func_data, dict) and 'func' in func_data:
                    self.current_functions.append({
                        'name': func_name,
                        'data': func_data
                    })

            while True:
                print('\033[2J\033[H')

                print_box_header(f"Module: {self.current_module}", "📦")

                # Module info
                try:
                    mod = self.app.get_mod(self.current_module)
                    version = getattr(mod, 'version', 'unknown')
                    print_box_content(f"Version: {version}", "info")
                except:
                    print_box_content("Version: unknown", "warning")

                print_box_content(f"Functions: {len(self.current_functions)}", "info")
                print_box_footer()
                print()

                if not self.current_functions:
                    print_status("No functions available in this module", "warning")
                    print()
                    print_status("Press any key to go back...", "info")
                    get_key()
                    self.go_back()
                    return

                # Show functions
                visible_count = 12
                start_idx = max(0, self.selected_index - visible_count // 2)
                end_idx = min(len(self.current_functions), start_idx + visible_count)

                if end_idx - start_idx < visible_count:
                    start_idx = max(0, end_idx - visible_count)

                for i in range(start_idx, end_idx):
                    func = self.current_functions[i]
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    # Get function type
                    func_type = func['data'].get('type', 'unknown')
                    type_icon = "⚡" if 'async' in str(func_type) else "🔧"

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{type_icon} {func['name']}\033[0m")
                    else:
                        print(f"  {arrow} {type_icon} {func['name']}")

                if len(self.current_functions) > visible_count:
                    print(f"\n  Showing {start_idx + 1}-{end_idx} of {len(self.current_functions)}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Execute | i: Info | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(self.current_functions) - 1, self.selected_index + 1)
                elif key == 'enter':
                    self.history.append(self.current_view)
                    self.current_view = "function_execute"
                    return
                elif key in ('i', 'I'):
                    await self.show_function_info(self.current_functions[self.selected_index])

        async def show_function_runner(self):
            """Interactive function runner with autocomplete"""
            print('\033[2J\033[H')

            print_box_header("Function Runner", "🎯")
            print_box_content("Execute functions with autocomplete", "info")
            print_box_footer()

            # Restore terminal for input
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            print()
            print("  Format: module_name function_name [args...]")
            print("  Example: CloudM Version")
            print("  Example: helper create-user john john@mail.com")
            print()

            # Get all available modules and functions for autocomplete hints
            available_modules = list(self.app.functions.keys())

            print("  Available modules:")
            print("  " + ", ".join(available_modules[:10]))
            if len(available_modules) > 10:
                print(f"  ... and {len(available_modules) - 10} more")
            print()

            command_input = input("  Command: ").strip()

            if not command_input:
                self.go_back()
                return

            parts = command_input.split()

            if len(parts) < 2:
                print()
                print_status("Need at least module and function name", "error")
                print_status("Press any key to continue...", "info")
                get_key()
                return

            module_name = parts[0]
            function_name = parts[1]
            args = parts[2:]

            # Check if module exists
            if module_name not in self.app.functions:
                print()
                print_status(f"Module '{module_name}' not found", "error")

                # Suggest similar modules
                similar = [m for m in available_modules if module_name.lower() in m.lower()]
                if similar:
                    print()
                    print("  Did you mean:")
                    for s in similar[:5]:
                        print(f"    • {s}")

                print()
                print_status("Press any key to continue...", "info")
                get_key()
                return

            # Check if function exists
            module_data = self.app.functions.get(module_name, {})
            if function_name not in module_data:
                print()
                print_status(f"Function '{function_name}' not found in {module_name}", "error")

                # Show available functions
                available_funcs = [f for f in module_data.keys() if isinstance(module_data[f], dict)]
                if available_funcs:
                    print()
                    print("  Available functions:")
                    for f in available_funcs[:10]:
                        print(f"    • {f}")
                    if len(available_funcs) > 10:
                        print(f"    ... and {len(available_funcs) - 10} more")

                print()
                print_status("Press any key to continue...", "info")
                get_key()
                return

            # Ask for kwargs
            print()
            print_status("Enter keyword arguments (optional)", "info")
            kwargs_input = input("  Kwargs (key=value, space-separated): ").strip()

            kwargs = {}
            if kwargs_input:
                for pair in kwargs_input.split():
                    if '=' in pair:
                        key, value = pair.split('=', 1)
                        kwargs[key.strip()] = value.strip()
                    elif ':' in pair:
                        key, value = pair.split(':', 1)
                        kwargs[key.strip()] = value.strip()

            print()
            print_separator("═")
            print(f"  Executing: {module_name}.{function_name}")
            print_separator("═")
            print()

            try:
                # Execute function
                result = await self.app.a_run_any(
                    (module_name, function_name),
                    args_=args,
                    tb_run_with_specification='app',
                    get_results=True,
                    **kwargs
                )

                # Handle coroutine results
                if asyncio.iscoroutine(result):
                    result = await result

                if isinstance(result, asyncio.Task):
                    result = await result

                print()
                print_separator("═")
                print("  Result:")
                print_separator("═")
                print()

                if hasattr(result, 'print'):
                    result.print(full_data=True)
                elif hasattr(result, '__dict__'):
                    import pprint
                    pprint.pprint(result.__dict__)
                else:
                    print(f"  {result}")

                print()
                print_status("Execution completed successfully", "success")

            except Exception as e:
                print()
                print_status(f"Execution failed: {e}", "error")

                import traceback
                print()
                print("  Traceback:")
                print_separator()
                traceback.print_exc()

            print()
            print_status("Press any key to continue...", "info")
            get_key()

            # Ask if user wants to run another command
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            again = input("\n  Run another command? (y/N): ").strip().lower()

            if again == 'y':
                # Stay in function runner
                return
            else:
                self.go_back()

        async def show_workflow_runner(self):
            """Interactive workflow runner with autocomplete"""
            print('\033[2J\033[H')

            print_box_header("Workflow Runner", "🎯")
            print_box_content("Execute workflows with autocomplete", "info")
            print_box_footer()

            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            all_flows = self.app.flows.keys()
            if not all_flows:
                from toolboxv2.flows import flows_dict as flows_dict_func
                flows_dict = flows_dict_func(remote=False)
                self.app.set_flows(flows_dict)
                all_flows = self.app.flows.keys()
            print("  Available workflows:")
            # show in an 3 by n grid
            for i, flow in enumerate(all_flows):
                print(f" {str(i) + ' '+flow:<20}", end='\n' if i % 3 == 2 else ' ')

            command_input = input("  Workflow: ").strip()

            try:
                command_input = int(command_input)
                command_input = list(all_flows)[command_input]
            except:
                pass

            if not command_input:
                self.go_back()
                return

            if command_input not in all_flows:
                print()
                print_status(f"Workflow '{command_input}' not found", "error")
                print_status("Press any key to continue...", "info")
                get_key()
                return

            print()
            print_separator("═")
            print(f"  Executing: {command_input}")
            print_separator("═")
            print()
            try:
                self.go_back()
                await self.app.run_flows(command_input)
                print()
                print_status("Execution completed successfully", "success")
            except Exception as e:
                print()
                print_status(f"Execution failed: {e}", "error")
                import traceback
                print()
                print("  Traceback:")
                print_separator()
                traceback.print_exc()


        async def execute_function(self):
            """Execute selected function"""
            if not self.current_module or not self.current_functions:
                self.go_back()
                return

            func = self.current_functions[self.selected_index]

            print('\033[2J\033[H')

            print_box_header(f"Execute Function", "⚡")
            print_box_content(f"Module: {self.current_module}", "info")
            print_box_content(f"Function: {func['name']}", "info")
            print_box_footer()

            # Restore terminal for input
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            # Get arguments
            print()
            print_status("Enter function arguments (leave empty if none)", "info")
            args_input = input("  Args (space-separated): ").strip()

            args = args_input.split() if args_input else []

            print()
            print_status("Enter keyword arguments (leave empty if none)", "info")
            kwargs_input = input("  Kwargs (key=value, space-separated): ").strip()

            kwargs = {}
            if kwargs_input:
                for pair in kwargs_input.split():
                    if '=' in pair:
                        key, value = pair.split('=', 1)
                        kwargs[key.strip()] = value.strip()

            print()
            print_separator("═")
            print("  Executing...")
            print_separator("═")
            print()

            try:
                # Execute
                result = await self.app.a_run_any(
                    (self.current_module, func['name']),
                    args_=args,
                    tb_run_with_specification='app',
                    get_results=True,
                    **kwargs
                )

                if asyncio.iscoroutine(result):
                    result = await result

                print()
                print_separator("═")
                print("  Result:")
                print_separator("═")
                print()

                if hasattr(result, 'print'):
                    result.print(full_data=True)
                else:
                    print(f"  {result}")

                print()
                print_status("Execution completed successfully", "success")

            except Exception as e:
                print()
                print_status(f"Execution failed: {e}", "error")

                import traceback
                print()
                print("  Traceback:")
                print_separator()
                print(traceback.format_exc())

            print()
            print_status("Press any key to continue...", "info")
            get_key()

            self.go_back()

        async def show_status(self):
            """Show system status"""
            print('\033[2J\033[H')

            print_box_header("System Status", "📊")

            # User info
            try:
                username = self.app.get_username() if hasattr(self.app, 'get_username') else "Guest"
                login_status = "Not logged in"
                login_style = "error"

                # Check login status
                try:
                    from toolboxv2.utils.extras.blobs import BlobFile
                    from toolboxv2.utils.security.cryp import Code

                    with BlobFile(f"claim/{username}/jwt.c", key=Code.DK()(), mode="r") as blob:
                        claim = blob.read()
                        if claim and claim != b'Error decoding':
                            login_status = "Logged in"
                            login_style = "success"
                except:
                    pass
            except:
                username = "Guest"
                login_status = "Not logged in"
                login_style = "error"

            print_box_content(f"User: {username}", "info")
            print_box_content(f"Status: {login_status}", login_style)
            print_box_content(f"Instance: {self.app.id}", "info")
            print_box_content(f"System: {system()} on {node()}", "info")

            # Modules
            modules_count = len(self.app.functions.keys())
            print_box_content(f"Loaded Modules: {modules_count}", "info")

            # Services Status
            print_separator("─")

            # Check DB
            db_status = "Available"
            db_style = "success"
            try:
                # Quick check without full status output
                pass
            except:
                db_status = "Not available"
                db_style = "error"

            print_box_content(f"Database: {db_status}", db_style)

            # Check API
            api_status = "Stopped"
            api_style = "error"
            api_info = ""
            try:
                from toolboxv2.utils.system.state_system import read_server_state
                pid, _, _ = read_server_state()
                from toolboxv2.utils.system.state_system import is_process_running
                if is_process_running(pid):
                    api_status = "Running"
                    api_style = "success"
                    api_info = f" (PID: {pid})"
            except:
                pass

            print_box_content(f"API Server: {api_status}{api_info}", api_style)

            print_box_footer()

            print_status("Press any key to go back...", "info")
            get_key()

            self.go_back()

        async def show_services(self):
            """Show services management"""
            services = [
                ("🖥️", "API Server", "api"),
                ("🗄️", "Database", "db"),
                ("🌐", "P2P Client", "p2p"),
                ("📦", "Module Manager", "mods"),
                ("🔙", "Back", "back")
            ]

            while True:
                print('\033[2J\033[H')

                print_box_header("Service Management", "🔧")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(services):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Manage | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(services) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action = services[self.selected_index]

                    if action == "back":
                        self.go_back()
                        return
                    else:
                        await self.manage_service(action)

        async def manage_service(self, service_name: str):
            """Manage a specific service"""
            print('\033[2J\033[H')

            print_box_header(f"Manage {service_name.upper()}", "🔧")
            print_box_footer()

            actions = [
                ("▶️", "Start", "start"),
                ("⏹️", "Stop", "stop"),
                ("📊", "Status", "status"),
            ]

            if service_name == "api":
                # build, debug, clean, remove-exe, update
                actions.extend([
                    ("🔨", "Build", "build"),
                    ("🔥", "Debug", "debug"),
                    ("🧹", "Clean", "clean"),
                    ("🗑️", "Remove Executable", "remove-exe"),
                    ("🔄", "Update", "update"),
                ])
            if service_name == "db":
                # health, update , build, clean, discover
                actions.extend([
                    ("❤️", "Health", "health"),
                    ("🔄", "Update", "update"),
                    ("🔨", "Build", "build"),
                    ("🧹", "Clean", "clean"),
                    ("🔍", "Discover", "discover"),
                ])
            if service_name == "p2p":
                # interactive
                actions.append(("🎮", "Interactive", "interactive"))
                actions.remove(("▶️", "Start", "start"))
                actions.remove(("⏹️", "Stop", "stop"))

            actions.append(("🔙", "Back", "back"))

            action_idx = 0

            while True:
                print('\033[2J\033[H')

                print_box_header(f"Manage {service_name.upper()}", "🔧")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(actions):
                    is_selected = i == action_idx
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Execute | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    return
                elif key == 'up':
                    action_idx = max(0, action_idx - 1)
                elif key == 'down':
                    action_idx = min(len(actions) - 1, action_idx + 1)
                elif key == 'enter':
                    _, _, action = actions[action_idx]

                    if action == "back":
                        return

                    print()
                    print_separator("═")
                    print(f"  Executing: {action} on {service_name}")
                    print_separator("═")
                    print()

                    # Execute action
                    try:
                        if service_name == "api":
                            sys.argv = ["api", action]
                            cli_api_runner()
                        elif service_name == "db":
                            sys.argv = ["db", action]
                            cli_db_runner()
                        elif service_name == "p2p":
                            sys.argv = ["p2p", action]
                            cli_tcm_runner()
                        elif service_name == "mods":
                            await self.app.a_run_any("CloudM", "manager")

                        print()
                        print_status("Command executed", "success")
                    except Exception as e:
                        print()
                        print_status(f"Error: {e}", "error")

                    print()
                    print_status("Press any key to continue...", "info")
                    get_key()

        async def show_quick_actions(self):
            """Show quick actions menu"""
            actions = [
                ("🔐", "Login", self.quick_login),
                ("🚪", "Logout", self.quick_logout),
                ("📊", "System Status", self.quick_status),
                ("🔄", "Reload Modules", self.quick_reload),
                ("🧹", "Clear Cache", self.quick_clear_cache),
                ("🔙", "Back", None)
            ]

            while True:
                print('\033[2J\033[H')

                print_box_header("Quick Actions", "⚡")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(actions):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Execute | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(actions) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action_func = actions[self.selected_index]

                    if action_func is None:
                        self.go_back()
                        return

                    await action_func()

        async def show_search(self):
            """Show search interface"""
            print('\033[2J\033[H')

            print_box_header("Search", "🔍")
            print_box_footer()

            # Restore terminal for input
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            print()
            query = input("  Search query: ").strip().lower()

            if not query:
                self.go_back()
                return

            print()
            print_status("Searching...", "progress")

            # Search in modules and functions
            results = []

            for module_name in self.app.functions.keys():
                if query in module_name.lower():
                    results.append(("module", module_name, None))

                module_data = self.app.functions.get(module_name, {})
                for func_name in module_data.keys():
                    if query in func_name.lower():
                        results.append(("function", module_name, func_name))

            print('\033[2J\033[H')

            print_box_header(f"Search Results for '{query}'", "🔍")
            print_box_content(f"Found {len(results)} results", "info")
            print_box_footer()

            if not results:
                print()
                print_status("No results found", "warning")
            else:
                print()
                for result_type, module, func in results[:20]:  # Limit to 20 results
                    if result_type == "module":
                        print(f"  📦 Module: {module}")
                    else:
                        print(f"  ⚡ Function: {module}.{func}")

                if len(results) > 20:
                    print(f"\n  ... and {len(results) - 20} more results")

            print()
            print_separator()
            print_status("Press any key to go back...", "info")
            get_key()

            self.go_back()

        async def show_settings(self):
            """Show settings menu"""
            settings = [
                ("🔧", "Environment Variables", "env"),
                ("📝", "View Config", "view_config"),
                ("💾", "Save Config", "save_config"),
                ("📈", "App Footprint", "app_footprint"),
                ("ℹ️", "About", "about"),

                ("🔙", "Back", "back")
            ]

            self.selected_index = 0

            while True:
                print('\033[2J\033[H')

                print_box_header("Settings", "⚙️")
                print_box_footer()

                print()
                for i, (icon, label, _) in enumerate(settings):
                    is_selected = i == self.selected_index
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Open | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    self.go_back()
                    return
                elif key == 'up':
                    self.selected_index = max(0, self.selected_index - 1)
                elif key == 'down':
                    self.selected_index = min(len(settings) - 1, self.selected_index + 1)
                elif key == 'enter':
                    _, _, action = settings[self.selected_index]

                    if action == "back":
                        self.go_back()
                        return
                    elif action == "about":
                        await self.show_about()
                    elif action == "env":
                        await self.manage_env_vars()
                    elif action == "view_config":
                        await self.view_config()
                    elif action == "save_config":
                        await self.save_config()
                    elif action == "app_footprint":
                        print(get_app().print_footprint())
                        input(Style.GREY("Press Enter to continue..."))

        async def manage_env_vars(self):
            """Manage environment variables"""
            import os

            # Important ToolBox env vars
            env_vars = [
                ("TOOLBOXV2_REMOTE_BASE", "Remote server base URL", os.getenv("TOOLBOXV2_REMOTE_BASE", "")),
                ("APP_BASE_URL", "Application base URL", os.getenv("APP_BASE_URL", "")),
                ("TB_R_KEY", "Remote access key", os.getenv("TB_R_KEY", "")),
                ("DB_MODE_KEY", "Database mode", os.getenv("DB_MODE_KEY", "LC")),
                ("PYTHON_EXECUTABLE", "Python executable path", os.getenv("PYTHON_EXECUTABLE", "")),
                ("RUST_LOG", "Rust log level", os.getenv("RUST_LOG", "")),
            ]

            actions = [
                ("➕", "Add/Edit Variable", "edit"),
                ("📋", "View All", "view"),
                ("💾", "Save to .env", "save"),
                ("🔄", "Reload from .env", "reload"),
                ("🔙", "Back", "back")
            ]

            selected = 0

            while True:
                print('\033[2J\033[H')

                print_box_header("Environment Variables", "🔧")

                # Build ENV format string for display
                env_content = ""
                for var_name, description, value in env_vars:
                    env_content += f"# {description}\n"
                    if value:
                        env_content += f"{var_name}={value}\n"
                    else:
                        env_content += f"# {var_name}=(not set)\n"
                    env_content += "\n"

                # Display as formatted ENV file
                print_code_block(env_content.strip(), "env", show_line_numbers=False)
                print_box_footer()

                print()
                print_separator("─")
                print("  Actions:")
                print_separator("─")
                print()

                for i, (icon, label, _) in enumerate(actions):
                    is_selected = i == selected
                    arrow = "▶" if is_selected else " "

                    if is_selected:
                        print(f"  {arrow} \033[1;96m{icon} {label}\033[0m")
                    else:
                        print(f"  {arrow} {icon} {label}")

                print()
                print_separator()
                print_status("↑↓/w/s: Navigate | Enter: Select | b: Back", "info")

                key = get_key()

                if key in ('quit', 'b', 'B'):
                    return
                elif key == 'up':
                    selected = max(0, selected - 1)
                elif key == 'down':
                    selected = min(len(actions) - 1, selected + 1)
                elif key == 'enter':
                    _, _, action = actions[selected]

                    if action == "back":
                        return
                    elif action == "edit":
                        await self.edit_env_var(env_vars)
                    elif action == "view":
                        await self.view_all_env_vars()
                    elif action == "save":
                        await self.save_env_to_file(env_vars)
                    elif action == "reload":
                        await self.reload_env_from_file()

        async def edit_env_var(self, env_vars):
            """Edit an environment variable"""
            print('\033[2J\033[H')

            print_box_header("Edit Environment Variable", "✎")
            print_box_footer()

            # Restore terminal
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            print()
            print("  Available variables:")
            for i, (name, desc, _) in enumerate(env_vars, 1):
                print(f"    {i}. {name} - {desc}")

            print()
            choice = input("  Select variable number (or enter custom name): ").strip()

            try:
                idx = int(choice) - 1
                if 0 <= idx < len(env_vars):
                    var_name = env_vars[idx][0]
                else:
                    var_name = choice
            except ValueError:
                var_name = choice

            if not var_name:
                return

            current_value = os.getenv(var_name, "")
            print(f"\n  Current value: {current_value or '(not set)'}")

            new_value = input(f"  New value (leave empty to keep current): ").strip()

            if new_value:
                os.environ[var_name] = new_value
                print()
                print_status(f"Set {var_name} = {new_value}", "success")
            else:
                print()
                print_status("No changes made", "info")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def view_all_env_vars(self):
            """View all environment variables"""
            print('\033[2J\033[H')

            print_box_header("All Environment Variables", "📋")
            print_box_footer()

            env_vars = sorted(os.environ.items())

            print()
            print(f"  Total: {len(env_vars)} variables")
            print()
            print_separator()

            # Show first 30
            for key, value in env_vars[:30]:
                display_value = value
                if len(display_value) > 50:
                    display_value = display_value[:47] + "..."
                print(f"  {key:<30} = {display_value}")

            if len(env_vars) > 30:
                print(f"\n  ... and {len(env_vars) - 30} more")

            print()
            print_separator()
            print_status("Press any key to go back...", "info")
            get_key()

        async def save_env_to_file(self, env_vars):
            """Save environment variables to .env file"""
            from pathlib import Path

            print('\033[2J\033[H')

            print_box_header("Save to .env File", "💾")
            print_box_footer()

            env_file = Path(".env")

            print()
            print(f"  File: {env_file.absolute()}")
            print()

            # Restore terminal
            if system() != "Windows":
                import termios
                fd = sys.stdin.fileno()
                old_settings = termios.tcgetattr(fd)
                termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

            confirm = input("  Save current values to .env? (y/N): ").strip().lower()

            if confirm == 'y':
                try:
                    with open(env_file, 'w') as f:
                        for var_name, description, value in env_vars:
                            current = os.getenv(var_name, value)
                            if current:
                                f.write(f"# {description}\n")
                                f.write(f"{var_name}={current}\n\n")

                    print()
                    print_status(f"Saved to {env_file}", "success")
                except Exception as e:
                    print()
                    print_status(f"Error saving: {e}", "error")
            else:
                print()
                print_status("Cancelled", "info")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def reload_env_from_file(self):
            """Reload environment variables from .env file"""
            from pathlib import Path
            from dotenv import load_dotenv

            print('\033[2J\033[H')

            print_box_header("Reload from .env", "🔄")
            print_box_footer()

            env_file = Path(".env")

            print()
            if not env_file.exists():
                print_status(f".env file not found: {env_file.absolute()}", "warning")
            else:
                try:
                    load_dotenv(override=True)
                    print_status("Environment variables reloaded", "success")
                except Exception as e:
                    print_status(f"Error reloading: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def view_config(self):
            """View current configuration"""
            import json
            print('\033[2J\033[H')

            print_box_header("Current Configuration", "📝")

            # Build configuration as JSON
            modules_count = len(self.app.functions.keys())
            config_data = {
                "application": {
                    "instance_id": self.app.id,
                    "start_directory": str(self.app.start_dir),
                    "system": system(),
                    "node": node()
                },
                "modules": {
                    "loaded_count": modules_count,
                    "names": sorted(list(self.app.functions.keys())[:10])  # Show first 10
                },
                "environment": {
                    "remote_base": os.getenv("TOOLBOXV2_REMOTE_BASE", "not set"),
                    "app_base_url": os.getenv("APP_BASE_URL", "not set"),
                    "db_mode": os.getenv("DB_MODE_KEY", "LC")
                }
            }

            # Display as formatted JSON
            config_json = json.dumps(config_data, indent=2)
            print_code_block(config_json, "json", show_line_numbers=True)

            if modules_count > 10:
                print_box_content(f"... and {modules_count - 10} more modules", "info")

            print_box_footer()

            print_status("Press any key to go back...", "info")
            get_key()

        async def save_config(self):
            """Save current configuration"""
            print('\033[2J\033[H')

            print_box_header("Save Configuration", "💾")
            print_box_footer()

            print()
            print_status("Configuration auto-saved", "success")
            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def show_about(self):
            """Show about information"""
            print('\033[2J\033[H')

            print_box_header("About ToolBoxV2", "ℹ️")

            version = get_version_from_pyproject()
            from toolboxv2 import tb_root_dir, init_cwd

            print_box_content("ToolBoxV2 Interactive Dashboard", "info")
            print_box_content(f"Version: {version}", "info")
            print_box_content(f"System: {system()}", "info")
            print_box_content(f"Python: {sys.version.split()[0]}", "info")
            print_separator("─")
            print_box_content(f"Home: {tb_root_dir}", "info")
            print_box_content(f"Start: {init_cwd}", "info")
            print(f"  A powerful, modular Python framework")
            print(f"  for building and managing tools.")
            print()
            print_box_footer()

            print_status("Press any key to go back...", "info")
            get_key()

        async def show_help(self):
            """Show help screen"""
            print('\033[2J\033[H')

            print_box_header("Keyboard Shortcuts", "❓")
            print()
            print("  Navigation:")
            print("    ↑/↓ or w/s     Navigate menu items")
            print("    Enter          Select/Execute")
            print("    b / Esc        Go back")
            print()
            print("  Global:")
            print("    /              Search")
            print("    h              Show help")
            print("    q              Quit")
            print()
            print("  Function Execution:")
            print("    i              Show function info")
            print("    Enter          Execute function")
            print()
            print_box_footer()

            print_status("Press any key to continue...", "info")
            get_key()

        async def show_function_info(self, func):
            """Show detailed function information"""
            import json
            print('\033[2J\033[H')

            print_box_header(f"Function Info: {func['name']}", "ℹ️")

            func_data = func['data']

            # Build function info as JSON
            info_data = {
                "name": func['name'],
                "module": self.current_module,
                "type": func_data.get('type', 'unknown'),
            }

            if 'version' in func_data:
                info_data['version'] = func_data['version']

            if 'test' in func_data:
                info_data['testable'] = func_data['test']

            # Display as formatted JSON
            info_json = json.dumps(info_data, indent=2)
            print_code_block(info_json, "json", show_line_numbers=False)

            # Try to get docstring
            try:
                func_obj = func_data.get('func')
                if func_obj and hasattr(func_obj, '__doc__') and func_obj.__doc__:
                    print_separator("─")
                    print_box_content("Description:", "info")
                    docstring = func_obj.__doc__.strip()
                    # Display docstring with auto-wrap
                    for line in docstring.split('\n'):
                        if line.strip():
                            print_box_content(line.strip(), auto_wrap=True)
            except:
                pass

            print_box_footer()

            print_status("Press any key to continue...", "info")
            get_key()

        # Quick action implementations

        async def quick_login(self):
            """Quick login action"""
            print('\033[2J\033[H')
            print_box_header("Quick Login", "🔐")
            print_box_footer()

            try:
                result = await self.app.a_run_any("CloudM", "cli_web_login")
                print()
                if result:
                    print_status("Login successful!", "success")
                else:
                    print_status("Login failed or cancelled", "warning")
            except Exception as e:
                print_status(f"Error: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def quick_logout(self):
            """Quick logout action"""
            print('\033[2J\033[H')
            print_box_header("Quick Logout", "🚪")
            print_box_footer()

            try:
                result = await self.app.a_run_any("CloudM", "cli_logout")
                print()
                print_status("Logout successful!", "success")
            except Exception as e:
                print_status(f"Error: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def quick_status(self):
            """Quick status check"""
            await self.show_status()

        async def quick_reload(self):
            """Quick module reload"""
            print('\033[2J\033[H')
            print_box_header("Reload Modules", "🔄")
            print_box_footer()

            print()
            print_status("Reloading modules...", "progress")

            try:
                await self.app.load_all_mods_in_file()
                self.modules_cache = None  # Clear cache
                print_status("Modules reloaded successfully!", "success")
            except Exception as e:
                print_status(f"Error: {e}", "error")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        async def quick_clear_cache(self):
            """Clear dashboard cache"""
            print('\033[2J\033[H')
            print_box_header("Clear Cache", "🧹")
            print_box_footer()

            print()
            self.modules_cache = None
            self.current_functions = []
            print_status("Cache cleared!", "success")

            print()
            print_status("Press any key to continue...", "info")
            get_key()

        # Helper methods

        def go_back(self):
            """Go back to previous view"""
            if self.history:
                self.current_view = self.history.pop()
                self.selected_index = 0
            else:
                self.current_view = "main_menu"
                self.selected_index = 0

        async def confirm_exit(self):
            """Confirm exit"""
            print('\033[2J\033[H')

            print_box_header("Confirm Exit", "❓")
            print_box_content("Are you sure you want to exit?", "warning")
            print_box_footer()

            print()
            print("  Press 'y' to confirm, any other key to cancel")

            key = get_key()
            return key in ('y', 'Y')

    # =================== Main Entry Point ===================

    async def run_dashboard():
        """Run the dashboard"""
        # Setup app
        app= get_app(from_="run_dashboard")

        # Create and run dashboard
        dashboard = DashboardManager(app)

        # Load modules if not already loaded
        if not app.functions or len(app.functions) == 0:
            print_status("No modules loaded. Use -l flag to load all modules.", "info")
            print_status("or in ui '⚡ Quick Actions' -> '🔄 Reload Modules' ", "info")

        await dashboard.run()

        # Cleanup
        print('\033[2J\033[H')
        print_box_header("Goodbye!", "👋")
        print_box_content("Thank you for using ToolBoxV2", "success")
        print_box_footer()

        if not app.called_exit[0]:
            await app.a_exit()

    # Run
    await run_dashboard()
venv_runner

Modern Package Manager Runner - Supporting conda, uv, and native Python

BasePackageManager

Base class for package managers.

Source code in toolboxv2/utils/clis/venv_runner.py
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
class BasePackageManager:
    """Base class for package managers."""

    def __init__(self, runner: CommandRunner):
        self.runner = runner

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        raise NotImplementedError

    def delete_env(self, env_name: str) -> bool:
        raise NotImplementedError

    def list_envs(self) -> List[str]:
        raise NotImplementedError

    def install_package(self, env_name: str, package: str) -> bool:
        raise NotImplementedError

    def update_package(self, env_name: str, package: str) -> bool:
        raise NotImplementedError

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        raise NotImplementedError

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        raise NotImplementedError
CommandRunner

Enhanced command runner with better output handling.

Source code in toolboxv2/utils/clis/venv_runner.py
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
class CommandRunner:
    """Enhanced command runner with better output handling."""

    def __init__(self, package_manager: PackageManager):
        self.pm = package_manager
        self.encoding = get_encoding()

    def run(self, command: str, live: bool = True, capture: bool = False) -> Tuple[bool, Optional[str]]:
        """
        Execute command with optional live output.

        Args:
            command: Command to execute
            live: Stream output in real-time
            capture: Capture and return output

        Returns:
            Tuple of (success, output)
        """
        print_status('running', f'Executing: {command}')

        if live and not capture:
            # Stream output live
            try:
                process = subprocess.Popen(
                    command,
                    shell=True,
                    stdout=sys.stdout,
                    stderr=sys.stderr,
                    text=True,
                    encoding=self.encoding,
                    errors='replace'
                )
                process.communicate()
                success = process.returncode == 0

                if success:
                    print_status('success', 'Command completed successfully')
                else:
                    print_status('error', f'Command failed with code {process.returncode}')

                return success, None
            except Exception as e:
                print_status('error', f'Execution error: {e}')
                return False, None

        else:
            # Capture output
            try:
                result = subprocess.run(
                    command,
                    shell=True,
                    check=True,
                    text=True,
                    capture_output=True,
                    encoding=self.encoding,
                    errors='replace'
                )
                print_status('success', 'Command completed')
                return True, result.stdout

            except subprocess.CalledProcessError as e:
                print_status('error', 'Command failed')
                if e.stdout:
                    print(f"\nOutput:\n{e.stdout}")
                if e.stderr:
                    print(f"\nError:\n{e.stderr}")
                return False, None

            except Exception as e:
                print_status('error', f'Execution error: {e}')
                return False, None
run(command, live=True, capture=False)

Execute command with optional live output.

Parameters:

Name Type Description Default
command str

Command to execute

required
live bool

Stream output in real-time

True
capture bool

Capture and return output

False

Returns:

Type Description
Tuple[bool, Optional[str]]

Tuple of (success, output)

Source code in toolboxv2/utils/clis/venv_runner.py
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
def run(self, command: str, live: bool = True, capture: bool = False) -> Tuple[bool, Optional[str]]:
    """
    Execute command with optional live output.

    Args:
        command: Command to execute
        live: Stream output in real-time
        capture: Capture and return output

    Returns:
        Tuple of (success, output)
    """
    print_status('running', f'Executing: {command}')

    if live and not capture:
        # Stream output live
        try:
            process = subprocess.Popen(
                command,
                shell=True,
                stdout=sys.stdout,
                stderr=sys.stderr,
                text=True,
                encoding=self.encoding,
                errors='replace'
            )
            process.communicate()
            success = process.returncode == 0

            if success:
                print_status('success', 'Command completed successfully')
            else:
                print_status('error', f'Command failed with code {process.returncode}')

            return success, None
        except Exception as e:
            print_status('error', f'Execution error: {e}')
            return False, None

    else:
        # Capture output
        try:
            result = subprocess.run(
                command,
                shell=True,
                check=True,
                text=True,
                capture_output=True,
                encoding=self.encoding,
                errors='replace'
            )
            print_status('success', 'Command completed')
            return True, result.stdout

        except subprocess.CalledProcessError as e:
            print_status('error', 'Command failed')
            if e.stdout:
                print(f"\nOutput:\n{e.stdout}")
            if e.stderr:
                print(f"\nError:\n{e.stderr}")
            return False, None

        except Exception as e:
            print_status('error', f'Execution error: {e}')
            return False, None
CondaManager

Conda package manager implementation.

Source code in toolboxv2/utils/clis/venv_runner.py
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
class CondaManager(BasePackageManager):
    """Conda package manager implementation."""

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        command = f"conda create -n {env_name} python={python_version} -y"
        return self.runner.run(command)[0]

    def delete_env(self, env_name: str) -> bool:
        command = f"conda env remove -n {env_name} -y"
        success = self.runner.run(command)[0]

        # Clean up registry
        registry_file = Path(f"{env_name}_registry.json")
        if registry_file.exists():
            registry_file.unlink()
            print_status('info', f'Removed registry file: {registry_file}')

        return success

    def list_envs(self) -> List[str]:
        command = "conda env list --json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                data = json.loads(output)
                envs = [Path(env).name for env in data.get('envs', [])]
                return envs
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse environment list')

        return []

    def install_package(self, env_name: str, package: str) -> bool:
        command = f"conda install -n {env_name} {package} -y"
        success = self.runner.run(command)[0]

        if success:
            self._update_registry(env_name, package)

        return success

    def update_package(self, env_name: str, package: str) -> bool:
        command = f"conda update -n {env_name} {package} -y"
        return self.runner.run(command)[0]

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        command = f"conda list -n {env_name} --json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                packages = json.loads(output)
                return [{"name": pkg["name"], "version": pkg["version"]} for pkg in packages]
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse package list')

        return []

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        if python:
            command = f"conda run -v --no-capture-output -n {env_name} python {script} {' '.join(args)}"
        else:
            command = f"conda run -v --no-capture-output -n {env_name} {script} {' '.join(args)}"

        return self.runner.run(command)[0]

    def _update_registry(self, env_name: str, package: str):
        """Update package registry."""
        registry_file = Path(f"{env_name}_registry.json")

        try:
            if registry_file.exists():
                with open(registry_file) as f:
                    registry = json.load(f)
            else:
                registry = []

            if package not in registry:
                registry.append(package)

            with open(registry_file, 'w') as f:
                json.dump(registry, f, indent=2)

            print_status('info', f'Updated registry: {registry_file}')

        except Exception as e:
            print_status('warning', f'Failed to update registry: {e}')
NativeManager

Native Python (venv + pip) manager implementation.

Source code in toolboxv2/utils/clis/venv_runner.py
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
class NativeManager(BasePackageManager):
    """Native Python (venv + pip) manager implementation."""

    def __init__(self, runner: CommandRunner):
        super().__init__(runner)
        self.envs_base = Path.home() / ".python_envs"
        self.envs_base.mkdir(exist_ok=True)

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        env_path = self.envs_base / env_name
        command = f"python{python_version} -m venv {env_path}"

        # Fallback to default python if version not available
        if not shutil.which(f"python{python_version}"):
            print_status('warning', f'Python {python_version} not found, using default python')
            command = f"python -m venv {env_path}"

        return self.runner.run(command)[0]

    def delete_env(self, env_name: str) -> bool:
        env_path = self.envs_base / env_name

        if env_path.exists():
            try:
                shutil.rmtree(env_path)
                print_status('success', f'Removed environment: {env_path}')
                return True
            except Exception as e:
                print_status('error', f'Failed to remove environment: {e}')
                return False
        else:
            print_status('warning', f'Environment not found: {env_path}')
            return False

    def list_envs(self) -> List[str]:
        if self.envs_base.exists():
            return [d.name for d in self.envs_base.iterdir() if d.is_dir() and (d / "bin" / "python").exists()]
        return []

    def install_package(self, env_name: str, package: str) -> bool:
        env_path = self.envs_base / env_name
        pip_bin = env_path / "bin" / "pip"

        if sys.platform == "win32":
            pip_bin = env_path / "Scripts" / "pip.exe"

        command = f"{pip_bin} install {package}"
        return self.runner.run(command)[0]

    def update_package(self, env_name: str, package: str) -> bool:
        env_path = self.envs_base / env_name
        pip_bin = env_path / "bin" / "pip"

        if sys.platform == "win32":
            pip_bin = env_path / "Scripts" / "pip.exe"

        command = f"{pip_bin} install --upgrade {package}"
        return self.runner.run(command)[0]

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        env_path = self.envs_base / env_name
        pip_bin = env_path / "bin" / "pip"

        if sys.platform == "win32":
            pip_bin = env_path / "Scripts" / "pip.exe"

        command = f"{pip_bin} list --format json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                packages = json.loads(output)
                return [{"name": pkg["name"], "version": pkg["version"]} for pkg in packages]
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse package list')

        return []

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        env_path = self.envs_base / env_name
        python_bin = env_path / "bin" / "python"

        if sys.platform == "win32":
            python_bin = env_path / "Scripts" / "python.exe"

        if python:
            command = f"{python_bin} {script} {' '.join(args)}"
        else:
            command = f"{script} {' '.join(args)}"

        return self.runner.run(command)[0]
PackageManager

Supported package managers.

Source code in toolboxv2/utils/clis/venv_runner.py
24
25
26
27
28
class PackageManager(Enum):
    """Supported package managers."""
    CONDA = "conda"
    UV = "uv"
    NATIVE = "native"  # pip/venv
UVManager

UV package manager implementation.

Source code in toolboxv2/utils/clis/venv_runner.py
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
class UVManager(BasePackageManager):
    """UV package manager implementation."""

    def create_env(self, env_name: str, python_version: str = "3.11") -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv venv {env_path} --python {python_version}"
        return self.runner.run(command)[0]

    def delete_env(self, env_name: str) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name

        if env_path.exists():
            try:
                shutil.rmtree(env_path)
                print_status('success', f'Removed environment: {env_path}')
                return True
            except Exception as e:
                print_status('error', f'Failed to remove environment: {e}')
                return False
        else:
            print_status('warning', f'Environment not found: {env_path}')
            return False

    def list_envs(self) -> List[str]:
        envs_path = Path.home() / ".uv" / "envs"

        if envs_path.exists():
            return [d.name for d in envs_path.iterdir() if d.is_dir()]

        return []

    def install_package(self, env_name: str, package: str) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv pip install --python {env_path}/bin/python {package}"
        return self.runner.run(command)[0]

    def update_package(self, env_name: str, package: str) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv pip install --upgrade --python {env_path}/bin/python {package}"
        return self.runner.run(command)[0]

    def list_packages(self, env_name: str) -> List[Dict[str, str]]:
        env_path = Path.home() / ".uv" / "envs" / env_name
        command = f"uv pip list --python {env_path}/bin/python --format json"
        success, output = self.runner.run(command, live=False, capture=True)

        if success and output:
            try:
                packages = json.loads(output)
                return [{"name": pkg["name"], "version": pkg["version"]} for pkg in packages]
            except json.JSONDecodeError:
                print_status('error', 'Failed to parse package list')

        return []

    def run_script(self, env_name: str, script: str, args: List[str], python: bool = True) -> bool:
        env_path = Path.home() / ".uv" / "envs" / env_name
        python_bin = env_path / "bin" / "python"

        if python:
            command = f"{python_bin} {script} {' '.join(args)}"
        else:
            command = f"{script} {' '.join(args)}"

        return self.runner.run(command)[0]
create_manager(pm_type)

Create appropriate package manager.

Source code in toolboxv2/utils/clis/venv_runner.py
569
570
571
572
573
574
575
576
577
578
def create_manager(pm_type: PackageManager) -> BasePackageManager:
    """Create appropriate package manager."""
    runner = CommandRunner(pm_type)

    if pm_type == PackageManager.CONDA:
        return CondaManager(runner)
    elif pm_type == PackageManager.UV:
        return UVManager(runner)
    else:
        return NativeManager(runner)
create_parser()

Create modern CLI parser.

Source code in toolboxv2/utils/clis/venv_runner.py
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
def create_parser() -> argparse.ArgumentParser:
    """Create modern CLI parser."""

    parser = argparse.ArgumentParser(
        prog='tb venv',
        description=textwrap.dedent("""
        ╔════════════════════════════════════════════════════════════════════╗
        ║          🐍 Modern Python Environment Manager 🐍                   ║
        ╚════════════════════════════════════════════════════════════════════╝

        Unified interface for conda, uv, and native Python environments.

        """),
        epilog=textwrap.dedent("""
        ┌─ EXAMPLES ─────────────────────────────────────────────────────────┐
        │                                                                    │
        │  Environment Management:                                           │
        │    $ tb venv create myenv                  # Create environment    │
        │    $ tb venv list                          # List environments     │
        │    $ tb venv delete myenv                  # Delete environment    │
        │                                                                    │
        │  Package Management:                                               │
        │    $ tb venv install myenv numpy           # Install package       │
        │    $ tb venv update myenv numpy            # Update package        │
        │    $ tb venv packages myenv                # List packages         │
        │                                                                    │
        │  Script Execution:                                                 │
        │    $ tb venv run myenv script.py arg1      # Run Python script     │
        │    $ tb venv exec myenv command args       # Run command           │
        │                                                                    │
        │  Advanced:                                                         │
        │    $ tb venv registry myenv                # Create registry       │
        │    $ tb venv update-all myenv              # Update all packages   │
        │    $ tb venv --manager uv create myenv     # Use specific PM       │
        │                                                                    │
        └────────────────────────────────────────────────────────────────────┘
        """),
        formatter_class=argparse.RawDescriptionHelpFormatter
    )

    # Global options
    parser.add_argument('--manager', '-m',
                        choices=['conda', 'uv', 'native'],
                        help='Package manager to use (auto-detect if not specified)')

    parser.add_argument('--python', '-py',
                        default='3.11',
                        help='Python version (default: 3.11)')

    # Subcommands
    subparsers = parser.add_subparsers(dest='command', help='Available commands')

    # =================== ENVIRONMENT COMMANDS ===================

    # Create environment
    create_parser = subparsers.add_parser('create', help='Create new environment')
    create_parser.add_argument('env_name', help='Environment name')
    create_parser.add_argument('--python', '-py', help='Python version (default: 3.11)')

    # Delete environment
    delete_parser = subparsers.add_parser('delete', help='Delete environment')
    delete_parser.add_argument('env_name', help='Environment name')
    delete_parser.add_argument('--force', '-f', action='store_true', help='Skip confirmation')

    # List environments
    list_parser = subparsers.add_parser('list', help='List all environments')

    # =================== PACKAGE COMMANDS ===================

    # Install package
    install_parser = subparsers.add_parser('install', help='Install package')
    install_parser.add_argument('env_name', help='Environment name')
    install_parser.add_argument('packages', nargs='+', help='Package(s) to install')
    install_parser.add_argument('--save', '-s', action='store_true', help='Save to registry')

    # Update package
    update_parser = subparsers.add_parser('update', help='Update package')
    update_parser.add_argument('env_name', help='Environment name')
    update_parser.add_argument('package', nargs='?', help='Package to update (all if not specified)')

    # List packages
    packages_parser = subparsers.add_parser('packages', help='List installed packages')
    packages_parser.add_argument('env_name', help='Environment name')
    packages_parser.add_argument('--json', action='store_true', help='Output as JSON')

    # =================== EXECUTION COMMANDS ===================

    # Run Python script
    run_parser = subparsers.add_parser('run', help='Run Python script in environment')
    run_parser.add_argument('env_name', help='Environment name')
    run_parser.add_argument('script', help='Script to run')
    run_parser.add_argument('args', nargs='*', help='Script arguments')

    # Execute command
    exec_parser = subparsers.add_parser('exec', help='Execute command in environment')
    exec_parser.add_argument('env_name', help='Environment name')
    exec_parser.add_argument('command', help='Command to execute')
    exec_parser.add_argument('args', nargs='*', help='Command arguments')

    # =================== UTILITY COMMANDS ===================

    # Create registry
    registry_parser = subparsers.add_parser('registry', help='Create package registry')
    registry_parser.add_argument('env_name', help='Environment name')

    # Update all packages
    update_all_parser = subparsers.add_parser('update-all', help='Update all packages')
    update_all_parser.add_argument('env_name', help='Environment name')

    # Info command
    info_parser = subparsers.add_parser('info', help='Show environment information')
    info_parser.add_argument('env_name', nargs='?', help='Environment name (current if not specified)')
    # Discover environments
    discover_parser = subparsers.add_parser('discover', help='Discover existing environments from all managers')
    discover_parser.add_argument('--save', '-s', action='store_true', help='Save discovered environments to registry')
    discover_parser.add_argument('--json', action='store_true', help='Output as JSON')
    return parser
detect_package_manager()

Auto-detect available package manager.

Source code in toolboxv2/utils/clis/venv_runner.py
64
65
66
67
68
69
70
71
def detect_package_manager() -> PackageManager:
    """Auto-detect available package manager."""
    if shutil.which("uv"):
        return PackageManager.UV
    elif shutil.which("conda"):
        return PackageManager.CONDA
    else:
        return PackageManager.NATIVE
discover_environments()

Discover existing environments from all package managers.

Source code in toolboxv2/utils/clis/venv_runner.py
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
def discover_environments() -> Dict[str, List[Dict[str, str]]]:
    """Discover existing environments from all package managers."""
    from toolboxv2 import init_cwd
    discovered = {
        'conda': [],
        'uv': [],
        'native': []
    }

    # Discover Conda environments
    if shutil.which("conda"):
        try:
            result = subprocess.run(
                ["conda", "env", "list", "--json"],
                capture_output=True, text=True, check=True
            )
            data = json.loads(result.stdout)
            for env_path in data.get('envs', []):
                env_name = Path(env_path).name
                if env_name != 'base':  # Skip base environment
                    discovered['conda'].append({
                        'name': env_name,
                        'path': env_path,
                        'manager': 'conda'
                    })
        except (subprocess.CalledProcessError, json.JSONDecodeError):
            pass

    # Discover UV environments
    if shutil.which("uv"):
        uv_envs_path = Path.home() / ".uv" / "envs"
        if uv_envs_path.exists():
            for env_dir in uv_envs_path.iterdir():
                if env_dir.is_dir():
                    discovered['uv'].append({
                        'name': env_dir.name,
                        'path': str(env_dir),
                        'manager': 'uv'
                    })

    # Discover Native Python environments
    native_paths = [
        Path.home() / "python_env",
        Path.home() / ".python_envs",
        Path.cwd() / "venv",
        Path.cwd() / ".venv",
        Path.cwd() / "env",
        init_cwd / "python_env",
        init_cwd / ".python_envs",
        init_cwd/ "venv",
        init_cwd/ ".venv",
        init_cwd/ "env"
    ]

    for base_path in native_paths:
        if base_path.exists():
            if base_path.name in ['venv', '.venv', 'env']:
                # Single environment in current directory
                if _is_valid_venv(base_path):
                    discovered['native'].append({
                        'name': f"local-{base_path.name}",
                        'path': str(base_path),
                        'manager': 'native'
                    })
            else:
                # Multiple environments in directory
                for env_dir in base_path.iterdir():
                    if env_dir.is_dir() and _is_valid_venv(env_dir):
                        discovered['native'].append({
                            'name': env_dir.name,
                            'path': str(env_dir),
                            'manager': 'native'
                        })

    return discovered
get_encoding()

Get system encoding with fallback.

Source code in toolboxv2/utils/clis/venv_runner.py
74
75
76
77
78
79
def get_encoding():
    """Get system encoding with fallback."""
    try:
        return sys.stdout.encoding or 'utf-8'
    except:
        return 'utf-8'
handle_create(args, manager)

Handle environment creation.

Source code in toolboxv2/utils/clis/venv_runner.py
738
739
740
741
742
743
744
745
746
747
748
def handle_create(args, manager: BasePackageManager):
    """Handle environment creation."""
    print_header(f'Creating Environment: {args.env_name}')

    python_version = args.python or "3.11"

    if manager.create_env(args.env_name, python_version):
        print_status('success', f'Environment "{args.env_name}" created successfully!')
    else:
        print_status('error', f'Failed to create environment "{args.env_name}"')
        sys.exit(1)
handle_delete(args, manager)

Handle environment deletion.

Source code in toolboxv2/utils/clis/venv_runner.py
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
def handle_delete(args, manager: BasePackageManager):
    """Handle environment deletion."""
    if not args.force:
        # Confirm deletion
        result = yes_no_dialog(
            title='Confirm Deletion',
            text=f'Really delete environment "{args.env_name}"?\n\nThis action cannot be undone.',
            style=MODERN_STYLE
        ).run()

        if not result:
            print_status('info', 'Deletion cancelled')
            return

    print_header(f'Deleting Environment: {args.env_name}')

    if manager.delete_env(args.env_name):
        print_status('success', f'Environment "{args.env_name}" deleted successfully!')
    else:
        print_status('error', f'Failed to delete environment "{args.env_name}"')
        sys.exit(1)
handle_discover(args, manager)

Handle environment discovery.

Source code in toolboxv2/utils/clis/venv_runner.py
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
def handle_discover(args, manager: BasePackageManager):
    """Handle environment discovery."""
    print_header('Discovering Environments')

    discovered = discover_environments()

    total_found = sum(len(envs) for envs in discovered.values())

    if total_found == 0:
        print_status('warning', 'No environments discovered')
        return

    if args.json:
        print(json.dumps(discovered, indent=2))
        return

    # Display discovered environments
    for manager_name, envs in discovered.items():
        if envs:
            print(f"\n📦 {manager_name.upper()} Environments ({len(envs)} found):")
            print('─' * 60)

            for i, env in enumerate(envs, 1):
                print(f"  {i:>2}. {env['name']:<25}{env['path']}")

    print(f"\n🔍 Total discovered: {total_found} environment(s)")

    # Save to registry if requested
    if args.save:
        try:
            registry_file = save_discovered_environments(discovered)
            print_status('success', f'Environments saved to registry: {registry_file}')
        except Exception as e:
            print_status('error', f'Failed to save registry: {e}')
handle_exec(args, manager)

Handle command execution.

Source code in toolboxv2/utils/clis/venv_runner.py
865
866
867
868
869
870
871
872
873
def handle_exec(args, manager: BasePackageManager):
    """Handle command execution."""
    print_header(f'Executing Command in: {args.env_name}')

    if manager.run_script(args.env_name, args.command, args.args, python=False):
        print_status('success', 'Command completed successfully')
    else:
        print_status('error', 'Command execution failed')
        sys.exit(1)
handle_info(args, manager)

Handle info display.

Source code in toolboxv2/utils/clis/venv_runner.py
900
901
902
903
904
905
906
907
908
909
910
911
def handle_info(args, manager: BasePackageManager):
    """Handle info display."""
    env_name = args.env_name or 'current'

    print_header(f'Environment Info: {env_name}')

    # Show package count
    packages = manager.list_packages(env_name) if args.env_name else []

    print(f"Package Manager: {manager.runner.pm.value}")
    print(f"Total Packages: {len(packages)}")
    print()
handle_install(args, manager)

Handle package installation.

Source code in toolboxv2/utils/clis/venv_runner.py
793
794
795
796
797
798
799
800
801
802
803
def handle_install(args, manager: BasePackageManager):
    """Handle package installation."""
    print_header(f'Installing Packages in: {args.env_name}')

    for package in args.packages:
        print(f"\nInstalling {package}...")

        if manager.install_package(args.env_name, package):
            print_status('success', f'Package "{package}" installed')
        else:
            print_status('error', f'Failed to install "{package}"')
handle_list(args, manager)

Handle environment listing.

Source code in toolboxv2/utils/clis/venv_runner.py
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
def handle_list(args, manager: BasePackageManager):
    """Handle environment listing."""
    print_header('Available Environments')

    envs = manager.list_envs()

    if not envs:
        print_status('warning', 'No environments found')
        return

    print(f"\n{'#':<4} {'Environment Name':<30}")
    print('─' * 50)

    for i, env in enumerate(envs, 1):
        print(f"{i:<4} {env:<30}")

    print(f"\nTotal: {len(envs)} environment(s)\n")
handle_packages(args, manager)

Handle package listing.

Source code in toolboxv2/utils/clis/venv_runner.py
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
def handle_packages(args, manager: BasePackageManager):
    """Handle package listing."""
    print_header(f'Packages in: {args.env_name}')

    packages = manager.list_packages(args.env_name)

    if not packages:
        print_status('warning', 'No packages found')
        return

    if args.json:
        print(json.dumps(packages, indent=2))
    else:
        print(f"\n{'#':<6} {'Package':<35} {'Version':<15}")
        print('─' * 60)

        for i, pkg in enumerate(packages, 1):
            print(f"{i:<6} {pkg['name']:<35} {pkg['version']:<15}")

        print(f"\nTotal: {len(packages)} package(s)\n")
handle_registry(args, manager)

Handle registry creation.

Source code in toolboxv2/utils/clis/venv_runner.py
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
def handle_registry(args, manager: BasePackageManager):
    """Handle registry creation."""
    print_header(f'Creating Registry for: {args.env_name}')

    packages = manager.list_packages(args.env_name)

    if not packages:
        print_status('warning', 'No packages to register')
        return

    registry_file = Path(f"{args.env_name}_registry.json")

    try:
        with open(registry_file, 'w') as f:
            json.dump(packages, f, indent=2)

        print_status('success', f'Registry created: {registry_file}')
        print_status('info', f'Registered {len(packages)} package(s)')

    except Exception as e:
        print_status('error', f'Failed to create registry: {e}')
        sys.exit(1)
handle_run(args, manager)

Handle script execution.

Source code in toolboxv2/utils/clis/venv_runner.py
854
855
856
857
858
859
860
861
862
def handle_run(args, manager: BasePackageManager):
    """Handle script execution."""
    print_header(f'Running Script in: {args.env_name}')

    if manager.run_script(args.env_name, args.script, args.args, python=True):
        print_status('success', 'Script completed successfully')
    else:
        print_status('error', 'Script execution failed')
        sys.exit(1)
handle_update(args, manager)

Handle package update.

Source code in toolboxv2/utils/clis/venv_runner.py
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
def handle_update(args, manager: BasePackageManager):
    """Handle package update."""
    print_header(f'Updating Packages in: {args.env_name}')

    if args.package:
        # Update single package
        if manager.update_package(args.env_name, args.package):
            print_status('success', f'Package "{args.package}" updated')
        else:
            print_status('error', f'Failed to update "{args.package}"')
    else:
        # Update all packages
        packages = manager.list_packages(args.env_name)

        if not packages:
            print_status('warning', 'No packages found')
            return

        print(f"Updating {len(packages)} package(s)...")

        for pkg in tqdm(packages, desc="Updating"):
            manager.update_package(args.env_name, pkg['name'])

        print_status('success', 'All packages updated')
main()

Main entry point.

Source code in toolboxv2/utils/clis/venv_runner.py
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
def main():
    """Main entry point."""
    parser = create_parser()
    args = parser.parse_args()

    if not args.command:
        parser.print_help()
        return

    # Determine package manager
    if args.manager:
        pm_type = PackageManager(args.manager)
    else:
        pm_type = detect_package_manager()
        print_status('info', f'Auto-detected package manager: {pm_type.value}')

    # Create manager
    manager = create_manager(pm_type)

    # Handle command
    try:
        if args.command == 'create':
            handle_create(args, manager)
        elif args.command == 'delete':
            handle_delete(args, manager)
        elif args.command == 'list':
            handle_list(args, manager)
        elif args.command == 'install':
            handle_install(args, manager)
        elif args.command == 'update':
            handle_update(args, manager)
        elif args.command == 'packages':
            handle_packages(args, manager)
        elif args.command == 'run':
            handle_run(args, manager)
        elif args.command == 'exec':
            handle_exec(args, manager)
        elif args.command == 'registry':
            handle_registry(args, manager)
        elif args.command == 'update-all':
            handle_update(args, manager)
        elif args.command == 'info':
            handle_info(args, manager)
        elif args.command == 'discover':
            handle_discover(args, manager)

    except KeyboardInterrupt:
        print_status('warning', '\nOperation cancelled by user')
        sys.exit(130)

    except Exception as e:
        print_status('error', f'Unexpected error: {e}')
        import traceback
        traceback.print_exc()
        sys.exit(1)
print_header(title)

Print section header.

Source code in toolboxv2/utils/clis/venv_runner.py
56
57
58
59
60
61
def print_header(title: str):
    """Print section header."""
    width = 78
    print_formatted_text(HTML(f'\n<header>{"─" * width}</header>'))
    print_formatted_text(HTML(f'<header>{title.center(width)}</header>'))
    print_formatted_text(HTML(f'<header>{"─" * width}</header>\n'))
print_status(status, message)

Print colored status message.

Source code in toolboxv2/utils/clis/venv_runner.py
43
44
45
46
47
48
49
50
51
52
53
def print_status(status: str, message: str):
    """Print colored status message."""
    icons = {
        'success': '✓',
        'error': '✗',
        'warning': '⚠',
        'info': 'ℹ',
        'running': '⟳'
    }
    icon = icons.get(status, '•')
    print_formatted_text(HTML(f'<{status}>{icon} {message}</{status}>'), style=MODERN_STYLE)
save_discovered_environments(discovered)

Save discovered environments to registry file.

Source code in toolboxv2/utils/clis/venv_runner.py
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def save_discovered_environments(discovered: Dict[str, List[Dict[str, str]]]) -> Path:
    """Save discovered environments to registry file."""
    registry_file = Path.home() / ".toolbox_env_registry.json"

    # Load existing registry or create new
    existing_registry = {}
    if registry_file.exists():
        try:
            with open(registry_file, 'r') as f:
                existing_registry = json.load(f)
        except (json.JSONDecodeError, IOError):
            pass

    # Merge discovered environments
    for manager, envs in discovered.items():
        if manager not in existing_registry:
            existing_registry[manager] = []

        # Add new environments (avoid duplicates)
        existing_names = {env['name'] for env in existing_registry[manager]}
        for env in envs:
            if env['name'] not in existing_names:
                existing_registry[manager].append(env)

    # Save updated registry
    try:
        with open(registry_file, 'w') as f:
            json.dump(existing_registry, f, indent=2)
        return registry_file
    except IOError as e:
        raise Exception(f"Failed to save registry: {e}")

daemon

DaemonUtil
Source code in toolboxv2/utils/daemon/daemon_util.py
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
class DaemonUtil:

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.server = None
        self.alive = False
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, t=False,
                        app: (App or AppType) | None = None,
                        peer=False, name='daemonApp-server', on_register=None, on_client_exit=None, on_server_exit=None,
                        unix_socket=False, test_override=False):
        from toolboxv2.mods.SocketManager import SocketType
        self.class_instance = class_instance
        self.server = None
        self.port = port
        self.host = host
        self.alive = False
        self.test_override = test_override
        self._name = name
        if on_register is None:
            def on_register(*args):
                return None
        self._on_register = on_register
        if on_client_exit is None:
            def on_client_exit(*args):
                return None
        self.on_client_exit = on_client_exit
        if on_server_exit is None:
            def on_server_exit():
                return None
        self.on_server_exit = on_server_exit
        self.unix_socket = unix_socket
        self.online = None
        connection_type = SocketType.server
        if peer:
            connection_type = SocketType.peer

        await self.start_server(connection_type)
        app = app if app is not None else get_app(from_=f"DaemonUtil.{self._name}")
        self.online = await asyncio.to_thread(self.connect, app)
        if t:
            await self.online

    async def start_server(self, connection_type=None):
        """Start the server using app and the socket manager"""
        from toolboxv2.mods.SocketManager import SocketType
        if connection_type is None:
            connection_type = SocketType.server
        app = get_app(from_="Starting.Daemon")
        print(app.mod_online("SocketManager"), "SocketManager")
        if not app.mod_online("SocketManager"):
            await app.load_mod("SocketManager")
        server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                            get_results=True,
                                            name=self._name,
                                            host=self.host,
                                            port=self.port,
                                            type_id=connection_type,
                                            max_connections=-1,
                                            return_full_object=True,
                                            test_override=self.test_override,
                                            unix_file=self.unix_socket)
        if server_result.is_error():
            raise Exception(f"Server error: {server_result.print(False)}")
        if not server_result.is_data():
            raise Exception(f"Server error: {server_result.print(False)}")
        self.alive = True
        self.server = server_result
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,

    async def send(self, data: dict or bytes or str, identifier: tuple[str, int] or str = "main"):
        result = await self.server.aget()
        sender = result.get('sender')
        await sender(data, identifier)
        return "Data Transmitted"

    @staticmethod
    async def runner_co(fuction, *args, **kwargs):
        if asyncio.iscoroutinefunction(fuction):
            return await fuction(*args, **kwargs)
        return fuction(*args, **kwargs)

    async def connect(self, app):
        result = await self.server.aget()
        if not isinstance(result, dict) or result.get('connection_error') != 0:
            raise Exception(f"Server error: {result}")
        self.server = Result.ok(result)
        receiver_queue: queue.Queue = self.server.get('receiver_queue')
        client_to_receiver_thread = self.server.get('client_to_receiver_thread')
        running_dict = self.server.get('running_dict')
        sender = self.server.get('sender')
        known_clients = {}
        valid_clients = {}
        app.print(f"Starting Demon {self._name}")

        while self.alive:

            if not receiver_queue.empty():
                data = receiver_queue.get()
                print(data)
                if not data:
                    continue
                if 'identifier' not in data:
                    continue

                identifier = data.get('identifier', 'unknown')
                try:
                    if identifier == "new_con":
                        client, address = data.get('data')
                        get_logger().info(f"New connection: {address}")
                        known_clients[str(address)] = client
                        await client_to_receiver_thread(client, str(address))

                        await self.runner_co(self._on_register, identifier, address)
                        identifier = str(address)
                        # await sender({'ok': 0}, identifier)

                    print("Receiver queue", identifier, identifier in known_clients, identifier in valid_clients)
                    # validation
                    if identifier in known_clients:
                        get_logger().info(identifier)
                        if identifier.startswith("('127.0.0.1'"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        elif data.get("claim", False):
                            do = app.run_any(("CloudM.UserInstances", "validate_ws_id"),
                                             ws_id=data.get("claim"))[0]
                            get_logger().info(do)
                            if do:
                                valid_clients[identifier] = known_clients[identifier]
                                await self.runner_co(self._on_register, identifier, data)
                        elif data.get("key", False) == os.getenv("TB_R_KEY"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        else:
                            get_logger().warning(f"Validating Failed: {identifier}")
                            # sender({'Validating Failed': -1}, eval(identifier))
                        get_logger().info(f"Validating New: {identifier}")
                        del known_clients[identifier]

                    elif identifier in valid_clients:
                        get_logger().info(f"New valid Request: {identifier}")
                        name = data.get('name')
                        args = data.get('args')
                        kwargs = data.get('kwargs')
                        if not name:
                            continue

                        get_logger().info(f"Request data: {name=}{args=}{kwargs=}{identifier=}")

                        if name == 'exit_main':
                            self.alive = False
                            break

                        if name == 'show_console':
                            show_console(True)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'hide_console':
                            show_console(False)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'rrun_flow':
                            show_console(True)
                            runnner = self.class_instance.run_flow
                            threading.Thread(target=runnner, args=args, kwargs=kwargs, daemon=True).start()
                            await sender({'ok': 0}, identifier)
                            show_console(False)
                            continue

                        async def _helper_runner():
                            try:
                                attr_f = getattr(self.class_instance, name)

                                if asyncio.iscoroutinefunction(attr_f):
                                    res = await attr_f(*args, **kwargs)
                                else:
                                    res = attr_f(*args, **kwargs)

                                if res is None:
                                    res = {'data': res}
                                elif isinstance(res, Result):
                                    if asyncio.iscoroutine(res.get()) or isinstance(res.get(), asyncio.Task):
                                        res_ = await res.aget()
                                        res.result.data = res_
                                    res = json.loads(res.to_api_result().json())
                                elif isinstance(res, bytes | dict):
                                    pass
                                else:
                                    res = {'data': 'unsupported type', 'type': str(type(res))}

                                get_logger().info(f"sending response {res} {type(res)}")

                                await sender(res, identifier)
                            except Exception as e:
                                import traceback
                                print(traceback.format_exc())
                                await sender({"data": str(e)}, identifier)

                        await _helper_runner()
                    else:
                        print("Unknown connection data:", data)

                except Exception as e:
                    get_logger().warning(Style.RED(f"An error occurred on {identifier} {str(e)}"))
                    if identifier != "unknown":
                        running_dict["receive"][str(identifier)] = False
                        await self.runner_co(self.on_client_exit,  identifier)
            await asyncio.sleep(0.1)
        running_dict["server_receiver"] = False
        for x in running_dict["receive"]:
            running_dict["receive"][x] = False
        running_dict["keep_alive_var"] = False
        await self.runner_co(self.on_server_exit)
        app.print(f"Closing Demon {self._name}")
        return Result.ok()

    async def a_exit(self):
        result = await self.server.aget()
        await result.get("close")()
        self.alive = False
        if asyncio.iscoroutine(self.online):
            await self.online
        print("Connection result :", result.get("host"), result.get("port"),
              "total connections:", result.get("connections"))
__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/daemon/daemon_util.py
19
20
21
22
23
24
25
26
27
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.server = None
    self.alive = False
    self.__storedargs = args, kwargs
    self.async_initialized = False
__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/daemon/daemon_util.py
29
30
31
32
33
34
35
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self
start_server(connection_type=None) async

Start the server using app and the socket manager

Source code in toolboxv2/utils/daemon/daemon_util.py
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
async def start_server(self, connection_type=None):
    """Start the server using app and the socket manager"""
    from toolboxv2.mods.SocketManager import SocketType
    if connection_type is None:
        connection_type = SocketType.server
    app = get_app(from_="Starting.Daemon")
    print(app.mod_online("SocketManager"), "SocketManager")
    if not app.mod_online("SocketManager"):
        await app.load_mod("SocketManager")
    server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                        get_results=True,
                                        name=self._name,
                                        host=self.host,
                                        port=self.port,
                                        type_id=connection_type,
                                        max_connections=-1,
                                        return_full_object=True,
                                        test_override=self.test_override,
                                        unix_file=self.unix_socket)
    if server_result.is_error():
        raise Exception(f"Server error: {server_result.print(False)}")
    if not server_result.is_data():
        raise Exception(f"Server error: {server_result.print(False)}")
    self.alive = True
    self.server = server_result
daemon_util
DaemonUtil
Source code in toolboxv2/utils/daemon/daemon_util.py
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
class DaemonUtil:

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.server = None
        self.alive = False
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, t=False,
                        app: (App or AppType) | None = None,
                        peer=False, name='daemonApp-server', on_register=None, on_client_exit=None, on_server_exit=None,
                        unix_socket=False, test_override=False):
        from toolboxv2.mods.SocketManager import SocketType
        self.class_instance = class_instance
        self.server = None
        self.port = port
        self.host = host
        self.alive = False
        self.test_override = test_override
        self._name = name
        if on_register is None:
            def on_register(*args):
                return None
        self._on_register = on_register
        if on_client_exit is None:
            def on_client_exit(*args):
                return None
        self.on_client_exit = on_client_exit
        if on_server_exit is None:
            def on_server_exit():
                return None
        self.on_server_exit = on_server_exit
        self.unix_socket = unix_socket
        self.online = None
        connection_type = SocketType.server
        if peer:
            connection_type = SocketType.peer

        await self.start_server(connection_type)
        app = app if app is not None else get_app(from_=f"DaemonUtil.{self._name}")
        self.online = await asyncio.to_thread(self.connect, app)
        if t:
            await self.online

    async def start_server(self, connection_type=None):
        """Start the server using app and the socket manager"""
        from toolboxv2.mods.SocketManager import SocketType
        if connection_type is None:
            connection_type = SocketType.server
        app = get_app(from_="Starting.Daemon")
        print(app.mod_online("SocketManager"), "SocketManager")
        if not app.mod_online("SocketManager"):
            await app.load_mod("SocketManager")
        server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                            get_results=True,
                                            name=self._name,
                                            host=self.host,
                                            port=self.port,
                                            type_id=connection_type,
                                            max_connections=-1,
                                            return_full_object=True,
                                            test_override=self.test_override,
                                            unix_file=self.unix_socket)
        if server_result.is_error():
            raise Exception(f"Server error: {server_result.print(False)}")
        if not server_result.is_data():
            raise Exception(f"Server error: {server_result.print(False)}")
        self.alive = True
        self.server = server_result
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,

    async def send(self, data: dict or bytes or str, identifier: tuple[str, int] or str = "main"):
        result = await self.server.aget()
        sender = result.get('sender')
        await sender(data, identifier)
        return "Data Transmitted"

    @staticmethod
    async def runner_co(fuction, *args, **kwargs):
        if asyncio.iscoroutinefunction(fuction):
            return await fuction(*args, **kwargs)
        return fuction(*args, **kwargs)

    async def connect(self, app):
        result = await self.server.aget()
        if not isinstance(result, dict) or result.get('connection_error') != 0:
            raise Exception(f"Server error: {result}")
        self.server = Result.ok(result)
        receiver_queue: queue.Queue = self.server.get('receiver_queue')
        client_to_receiver_thread = self.server.get('client_to_receiver_thread')
        running_dict = self.server.get('running_dict')
        sender = self.server.get('sender')
        known_clients = {}
        valid_clients = {}
        app.print(f"Starting Demon {self._name}")

        while self.alive:

            if not receiver_queue.empty():
                data = receiver_queue.get()
                print(data)
                if not data:
                    continue
                if 'identifier' not in data:
                    continue

                identifier = data.get('identifier', 'unknown')
                try:
                    if identifier == "new_con":
                        client, address = data.get('data')
                        get_logger().info(f"New connection: {address}")
                        known_clients[str(address)] = client
                        await client_to_receiver_thread(client, str(address))

                        await self.runner_co(self._on_register, identifier, address)
                        identifier = str(address)
                        # await sender({'ok': 0}, identifier)

                    print("Receiver queue", identifier, identifier in known_clients, identifier in valid_clients)
                    # validation
                    if identifier in known_clients:
                        get_logger().info(identifier)
                        if identifier.startswith("('127.0.0.1'"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        elif data.get("claim", False):
                            do = app.run_any(("CloudM.UserInstances", "validate_ws_id"),
                                             ws_id=data.get("claim"))[0]
                            get_logger().info(do)
                            if do:
                                valid_clients[identifier] = known_clients[identifier]
                                await self.runner_co(self._on_register, identifier, data)
                        elif data.get("key", False) == os.getenv("TB_R_KEY"):
                            valid_clients[identifier] = known_clients[identifier]
                            await self.runner_co(self._on_register, identifier, data)
                        else:
                            get_logger().warning(f"Validating Failed: {identifier}")
                            # sender({'Validating Failed': -1}, eval(identifier))
                        get_logger().info(f"Validating New: {identifier}")
                        del known_clients[identifier]

                    elif identifier in valid_clients:
                        get_logger().info(f"New valid Request: {identifier}")
                        name = data.get('name')
                        args = data.get('args')
                        kwargs = data.get('kwargs')
                        if not name:
                            continue

                        get_logger().info(f"Request data: {name=}{args=}{kwargs=}{identifier=}")

                        if name == 'exit_main':
                            self.alive = False
                            break

                        if name == 'show_console':
                            show_console(True)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'hide_console':
                            show_console(False)
                            await sender({'ok': 0}, identifier)
                            continue

                        if name == 'rrun_flow':
                            show_console(True)
                            runnner = self.class_instance.run_flow
                            threading.Thread(target=runnner, args=args, kwargs=kwargs, daemon=True).start()
                            await sender({'ok': 0}, identifier)
                            show_console(False)
                            continue

                        async def _helper_runner():
                            try:
                                attr_f = getattr(self.class_instance, name)

                                if asyncio.iscoroutinefunction(attr_f):
                                    res = await attr_f(*args, **kwargs)
                                else:
                                    res = attr_f(*args, **kwargs)

                                if res is None:
                                    res = {'data': res}
                                elif isinstance(res, Result):
                                    if asyncio.iscoroutine(res.get()) or isinstance(res.get(), asyncio.Task):
                                        res_ = await res.aget()
                                        res.result.data = res_
                                    res = json.loads(res.to_api_result().json())
                                elif isinstance(res, bytes | dict):
                                    pass
                                else:
                                    res = {'data': 'unsupported type', 'type': str(type(res))}

                                get_logger().info(f"sending response {res} {type(res)}")

                                await sender(res, identifier)
                            except Exception as e:
                                import traceback
                                print(traceback.format_exc())
                                await sender({"data": str(e)}, identifier)

                        await _helper_runner()
                    else:
                        print("Unknown connection data:", data)

                except Exception as e:
                    get_logger().warning(Style.RED(f"An error occurred on {identifier} {str(e)}"))
                    if identifier != "unknown":
                        running_dict["receive"][str(identifier)] = False
                        await self.runner_co(self.on_client_exit,  identifier)
            await asyncio.sleep(0.1)
        running_dict["server_receiver"] = False
        for x in running_dict["receive"]:
            running_dict["receive"][x] = False
        running_dict["keep_alive_var"] = False
        await self.runner_co(self.on_server_exit)
        app.print(f"Closing Demon {self._name}")
        return Result.ok()

    async def a_exit(self):
        result = await self.server.aget()
        await result.get("close")()
        self.alive = False
        if asyncio.iscoroutine(self.online):
            await self.online
        print("Connection result :", result.get("host"), result.get("port"),
              "total connections:", result.get("connections"))
__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/daemon/daemon_util.py
19
20
21
22
23
24
25
26
27
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.server = None
    self.alive = False
    self.__storedargs = args, kwargs
    self.async_initialized = False
__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/daemon/daemon_util.py
29
30
31
32
33
34
35
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self
start_server(connection_type=None) async

Start the server using app and the socket manager

Source code in toolboxv2/utils/daemon/daemon_util.py
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
async def start_server(self, connection_type=None):
    """Start the server using app and the socket manager"""
    from toolboxv2.mods.SocketManager import SocketType
    if connection_type is None:
        connection_type = SocketType.server
    app = get_app(from_="Starting.Daemon")
    print(app.mod_online("SocketManager"), "SocketManager")
    if not app.mod_online("SocketManager"):
        await app.load_mod("SocketManager")
    server_result = await app.a_run_any(SOCKETMANAGER.CREATE_SOCKET,
                                        get_results=True,
                                        name=self._name,
                                        host=self.host,
                                        port=self.port,
                                        type_id=connection_type,
                                        max_connections=-1,
                                        return_full_object=True,
                                        test_override=self.test_override,
                                        unix_file=self.unix_socket)
    if server_result.is_error():
        raise Exception(f"Server error: {server_result.print(False)}")
    if not server_result.is_data():
        raise Exception(f"Server error: {server_result.print(False)}")
    self.alive = True
    self.server = server_result

extras

BaseWidget
Source code in toolboxv2/utils/extras/base_widget.py
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
class BaseWidget:
    def __init__(self, name: str):
        self.name = name
        self.openWidgetsIDs = {}
        self.onReload = []
        self.iframes = {}

    def register(self, app, fuction, version=None, name="get_widget", level=1, **kwargs):
        if version is None:
            version = app.version
        app.tb(mod_name=self.name, version=version, request_as_kwarg=True, level=level, api=True, name=name, **kwargs)(
            fuction)

    def modify_iterator(self, iterator, replace):
        """
        ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
        {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
        """

        for item in iterator:
            modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                             range(len(replace))}
            yield modified_item

    def register2reload(self, *functions):
        for fuction in functions:
            def x(r):
                return fuction(request=r)
            self.onReload.append(x)

    def reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = function()
        return c

    async def oa_reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = await function() if asyncio.iscoroutinefunction(function) else function()
        return c

    @staticmethod
    def get_a_group(asset_name, template=None, file_path=None, a_kwargs=None):
        if a_kwargs is None:
            raise ValueError("a_kwargs must be specified")
        return [{'name': asset_name,
                 'file_path': file_path,
                 'kwargs': a_kwargs
                 } if file_path is not None else {'name': asset_name,
                                                  'template': template,
                                                  'kwargs': a_kwargs
                                                  }]

    def group_generator(self, asset_name: str, iterator: iter, template=None, file_path=None, a_kwargs=None):
        groups = []
        work_kwargs = a_kwargs
        for _i, data in enumerate(iterator):
            if isinstance(data, dict):
                work_kwargs = {**a_kwargs, **data}
            groups.append(self.get_a_group(asset_name, template=template, file_path=file_path, a_kwargs=work_kwargs))
        return groups

    def asset_loder(self, app, name, asset_id, file_path=None, template=None, iterator=None, **kwargs):
        a_kwargs = {**{
            'root': f"/api/{self.name}",
            'WidgetID': asset_id},
                    **kwargs}
        asset_name = f"{name}-{asset_id}"
        if iterator is None:
            group = self.get_a_group(asset_name,
                                     template=template,
                                     file_path=file_path,
                                     a_kwargs=a_kwargs)
        else:
            group = self.group_generator(asset_name,
                                         iterator=iterator,
                                         template=template,
                                         file_path=file_path,
                                         a_kwargs=a_kwargs)

        asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                            group_name=self.name,
                            collection={'name': f"{asset_name}",
                                        'group': group},
                            get_results=True)
        if asset.is_error():
            app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
            asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                                group_name=self.name,
                                collection={'name': f"{self.name}-{asset_name}",
                                            'group': group},
                                get_results=True)
        return asset

    def generate_html(self, app, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        return app.run_any(MINIMALHTML.GENERATE_HTML,
                           group_name=self.name,
                           collection_name=f"{name}-{asset_id}")

    def load_widget(self, app, request, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
        self.reload(request)
        html_widget = self.generate_html(app, name, asset_id)
        return html_widget[0]['html_element']

    @staticmethod
    async def get_user_from_request(app, request):
        from toolboxv2.mods.CloudM import User
        if request is None:
            return User()
        return await get_current_user_from_request(app, request)

    @staticmethod
    def get_s_id(request):
        from ..system.types import Result
        if request is None:
            return Result.default_internal_error("No request specified")
        return Result.ok(request.session.get('ID', ''))

    def reload(self, request):
        [_(request) for _ in self.onReload]

    async def oa_reload(self, request):
        [_(request) if not asyncio.iscoroutinefunction(_) else await _(request) for _ in self.onReload]

    async def get_widget(self, request, **kwargs):
        raise NotImplementedError

    def hash_wrapper(self, _id, _salt=''):
        from ..security.cryp import Code
        return Code.one_way_hash(text=_id, salt=_salt, pepper=self.name)

    def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
        """
        Registriert einen iframe mit gegebener ID und Quelle

        Args:
            iframe_id: Eindeutige ID für den iframe
            src: URL oder Pfad zur Quelle des iframes
            width: Breite des iframes (default: "100%")
            height: Höhe des iframes (default: "500px")
            **kwargs: Weitere iframe-Attribute
        """
        iframe_config = {
            'src': src,
            'width': width,
            'height': height,
            **kwargs
        }
        self.iframes[iframe_id] = iframe_config

    def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
        """
        Erstellt ein Asset für einen registrierten iframe

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        if iframe_id not in self.iframes:
            raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

        if asset_id is None:
            asset_id = str(uuid.uuid4())[:4]

        iframe_config = self.iframes[iframe_id]
        iframe_template = """
        <iframe id="{iframe_id}"
                src="{src}"
                width="{width}"
                height="{height}"
                frameborder="0"
                {additional_attrs}></iframe>
        """.strip()

        # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
        known_attrs = {'src', 'width', 'height'}
        additional_attrs = ' '.join(
            f'{k}="{v}"' for k, v in iframe_config.items()
            if k not in known_attrs
        )

        iframe_html = iframe_template.format(
            iframe_id=iframe_id,
            src=iframe_config['src'],
            width=iframe_config['width'],
            height=iframe_config['height'],
            additional_attrs=additional_attrs
        )

        return self.asset_loder(
            app=app,
            name=f"iframe-{iframe_id}",
            asset_id=asset_id,
            template=iframe_html
        )

    def load_iframe(self, app, iframe_id: str, asset_id: str = None):
        """
        Lädt einen registrierten iframe und gibt das HTML-Element zurück

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        self.create_iframe_asset(app, iframe_id, asset_id)
        return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']
create_iframe_asset(app, iframe_id, asset_id=None)

Erstellt ein Asset für einen registrierten iframe

Parameters:

Name Type Description Default
app

App-Instanz

required
iframe_id str

ID des registrierten iframes

required
asset_id str

Optional, spezifische Asset-ID

None
Source code in toolboxv2/utils/extras/base_widget.py
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
    """
    Erstellt ein Asset für einen registrierten iframe

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    if iframe_id not in self.iframes:
        raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

    if asset_id is None:
        asset_id = str(uuid.uuid4())[:4]

    iframe_config = self.iframes[iframe_id]
    iframe_template = """
    <iframe id="{iframe_id}"
            src="{src}"
            width="{width}"
            height="{height}"
            frameborder="0"
            {additional_attrs}></iframe>
    """.strip()

    # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
    known_attrs = {'src', 'width', 'height'}
    additional_attrs = ' '.join(
        f'{k}="{v}"' for k, v in iframe_config.items()
        if k not in known_attrs
    )

    iframe_html = iframe_template.format(
        iframe_id=iframe_id,
        src=iframe_config['src'],
        width=iframe_config['width'],
        height=iframe_config['height'],
        additional_attrs=additional_attrs
    )

    return self.asset_loder(
        app=app,
        name=f"iframe-{iframe_id}",
        asset_id=asset_id,
        template=iframe_html
    )
load_iframe(app, iframe_id, asset_id=None)

Lädt einen registrierten iframe und gibt das HTML-Element zurück

Parameters:

Name Type Description Default
app

App-Instanz

required
iframe_id str

ID des registrierten iframes

required
asset_id str

Optional, spezifische Asset-ID

None
Source code in toolboxv2/utils/extras/base_widget.py
280
281
282
283
284
285
286
287
288
289
290
def load_iframe(self, app, iframe_id: str, asset_id: str = None):
    """
    Lädt einen registrierten iframe und gibt das HTML-Element zurück

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    self.create_iframe_asset(app, iframe_id, asset_id)
    return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']
modify_iterator(iterator, replace)

['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'}, {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]

Source code in toolboxv2/utils/extras/base_widget.py
 94
 95
 96
 97
 98
 99
100
101
102
103
def modify_iterator(self, iterator, replace):
    """
    ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
    {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
    """

    for item in iterator:
        modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                         range(len(replace))}
        yield modified_item
register_iframe(iframe_id, src, width='100%', height='500px', **kwargs)

Registriert einen iframe mit gegebener ID und Quelle

Parameters:

Name Type Description Default
iframe_id str

Eindeutige ID für den iframe

required
src str

URL oder Pfad zur Quelle des iframes

required
width str

Breite des iframes (default: "100%")

'100%'
height str

Höhe des iframes (default: "500px")

'500px'
**kwargs

Weitere iframe-Attribute

{}
Source code in toolboxv2/utils/extras/base_widget.py
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
    """
    Registriert einen iframe mit gegebener ID und Quelle

    Args:
        iframe_id: Eindeutige ID für den iframe
        src: URL oder Pfad zur Quelle des iframes
        width: Breite des iframes (default: "100%")
        height: Höhe des iframes (default: "500px")
        **kwargs: Weitere iframe-Attribute
    """
    iframe_config = {
        'src': src,
        'width': width,
        'height': height,
        **kwargs
    }
    self.iframes[iframe_id] = iframe_config
ask_question(title, message, yes_callback=None, no_callback=None, **kwargs)

Ask a yes/no question

Source code in toolboxv2/utils/extras/notification.py
817
818
819
820
821
822
823
824
825
826
827
828
829
830
def ask_question(title: str, message: str,
                 yes_callback: Callable = None,
                 no_callback: Callable = None, **kwargs) -> Optional[str]:
    """Ask a yes/no question"""
    notifier = create_notification_system()

    actions = [
        NotificationAction("yes", "Yes", yes_callback, is_default=True),
        NotificationAction("no", "No", no_callback)
    ]

    return notifier.show_notification(
        title, message, NotificationType.QUESTION, actions=actions, **kwargs
    )
quick_error(title, message, **kwargs)

Quick error notification

Source code in toolboxv2/utils/extras/notification.py
811
812
813
814
def quick_error(title: str, message: str, **kwargs):
    """Quick error notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.ERROR, **kwargs)
quick_info(title, message, **kwargs)

Quick info notification

Source code in toolboxv2/utils/extras/notification.py
793
794
795
796
def quick_info(title: str, message: str, **kwargs):
    """Quick info notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.INFO, **kwargs)
quick_success(title, message, **kwargs)

Quick success notification

Source code in toolboxv2/utils/extras/notification.py
799
800
801
802
def quick_success(title: str, message: str, **kwargs):
    """Quick success notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.SUCCESS, **kwargs)
quick_warning(title, message, **kwargs)

Quick warning notification

Source code in toolboxv2/utils/extras/notification.py
805
806
807
808
def quick_warning(title: str, message: str, **kwargs):
    """Quick warning notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.WARNING, **kwargs)
Style
Spinner

Enhanced Spinner with tqdm-like line rendering.

Source code in toolboxv2/utils/extras/Style.py
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
class Spinner:
    """
    Enhanced Spinner with tqdm-like line rendering.
    """
    SYMBOL_SETS = {
        "c": ["◐", "◓", "◑", "◒"],
        "b": ["▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"],
        "d": ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"],
        "w": ["🌍", "🌎", "🌏"],
        "s": ["🌀   ", " 🌀  ", "  🌀 ", "   🌀", "  🌀 ", " 🌀  "],
        "+": ["+", "x"],
        "t": ["✶", "✸", "✹", "✺", "✹", "✷"]
    }

    def __init__(
        self,
        message: str = "Loading...",
        delay: float = 0.1,
        symbols=None,
        count_down: bool = False,
        time_in_s: float = 0
    ):
        """Initialize spinner with flexible configuration."""
        # Resolve symbol set.
        if isinstance(symbols, str):
            symbols = self.SYMBOL_SETS.get(symbols, None)

        # Default symbols if not provided.
        if symbols is None:
            symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

        # Test mode symbol set.
        if 'unittest' in sys.argv[0]:
            symbols = ['#', '=', '-']

        self.spinner = itertools.cycle(symbols)
        self.delay = delay
        self.message = message
        self.running = False
        self.spinner_thread = None
        self.max_t = time_in_s
        self.contd = count_down

        # Rendering management.
        self._is_primary = False
        self._start_time = 0

        # Central manager.
        self.manager = SpinnerManager()

    def _generate_render_line(self):
        """Generate the primary render line."""
        current_time = time.time()
        if self.contd:
            remaining = max(0, self.max_t - (current_time - self._start_time))
            time_display = f"{remaining:.2f}"
        else:
            time_display = f"{current_time - self._start_time:.2f}"

        symbol = next(self.spinner)
        return f"{symbol} {self.message} | {time_display}"

    def _generate_secondary_info(self):
        """Generate secondary spinner info for additional spinners."""
        return f"{self.message}"

    def __enter__(self):
        """Start the spinner."""
        self.running = True
        self._start_time = time.time()
        self.manager.register_spinner(self)
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        """Stop the spinner."""
        self.running = False
        self.manager.unregister_spinner(self)
        # Clear the spinner's line if it was the primary spinner.
        if self._is_primary:
            sys.stdout.write("\r\033[K")
            sys.stdout.flush()
__enter__()

Start the spinner.

Source code in toolboxv2/utils/extras/Style.py
644
645
646
647
648
649
def __enter__(self):
    """Start the spinner."""
    self.running = True
    self._start_time = time.time()
    self.manager.register_spinner(self)
    return self
__exit__(exc_type, exc_value, exc_traceback)

Stop the spinner.

Source code in toolboxv2/utils/extras/Style.py
651
652
653
654
655
656
657
658
def __exit__(self, exc_type, exc_value, exc_traceback):
    """Stop the spinner."""
    self.running = False
    self.manager.unregister_spinner(self)
    # Clear the spinner's line if it was the primary spinner.
    if self._is_primary:
        sys.stdout.write("\r\033[K")
        sys.stdout.flush()
__init__(message='Loading...', delay=0.1, symbols=None, count_down=False, time_in_s=0)

Initialize spinner with flexible configuration.

Source code in toolboxv2/utils/extras/Style.py
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
def __init__(
    self,
    message: str = "Loading...",
    delay: float = 0.1,
    symbols=None,
    count_down: bool = False,
    time_in_s: float = 0
):
    """Initialize spinner with flexible configuration."""
    # Resolve symbol set.
    if isinstance(symbols, str):
        symbols = self.SYMBOL_SETS.get(symbols, None)

    # Default symbols if not provided.
    if symbols is None:
        symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

    # Test mode symbol set.
    if 'unittest' in sys.argv[0]:
        symbols = ['#', '=', '-']

    self.spinner = itertools.cycle(symbols)
    self.delay = delay
    self.message = message
    self.running = False
    self.spinner_thread = None
    self.max_t = time_in_s
    self.contd = count_down

    # Rendering management.
    self._is_primary = False
    self._start_time = 0

    # Central manager.
    self.manager = SpinnerManager()
SpinnerManager

Manages multiple spinners to ensure tqdm-like line rendering. Automatically captures SIGINT (Ctrl+C) to stop all spinners.

Source code in toolboxv2/utils/extras/Style.py
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
class SpinnerManager(metaclass=Singleton):
    """
    Manages multiple spinners to ensure tqdm-like line rendering.
    Automatically captures SIGINT (Ctrl+C) to stop all spinners.
    """
    _instance = None

    def __new__(cls):
        if not cls._instance:
            cls._instance = super().__new__(cls)
            cls._instance._init_manager()
        return cls._instance

    def _init_manager(self):
        """Initialize spinner management resources and register SIGINT handler."""
        self._spinners = []
        self._lock = threading.Lock()
        self._render_thread = None
        self._should_run = False
        try:
            signal.signal(signal.SIGINT, self._signal_handler)
        except ValueError:
            print("Spinner Manager not in the min Thread no signal possible")
            pass

    def _signal_handler(self, signum, frame):
        """Handle SIGINT by stopping all spinners gracefully."""
        with self._lock:
            for spinner in self._spinners:
                spinner.running = False
            self._spinners.clear()
        self._should_run = False
        sys.stdout.write("\r\033[K")  # Clear the spinner's line.
        sys.stdout.flush()
        sys.exit(0)

    def register_spinner(self, spinner):
        """Register a new spinner."""
        with self._lock:
            # First spinner defines the rendering line.
            if not self._spinners:
                spinner._is_primary = True
            self._spinners.append(spinner)
            # Start rendering if not already running.
            if not self._should_run:
                self._should_run = True
                self._render_thread = threading.Thread(
                    target=self._render_loop,
                    daemon=True
                )
                self._render_thread.start()

    def unregister_spinner(self, spinner):
        """Unregister a completed spinner."""
        with self._lock:
            if spinner in self._spinners:
                self._spinners.remove(spinner)

    def _render_loop(self):
        """Continuous rendering loop for all active spinners."""
        while self._should_run:
            if not self._spinners:
                self._should_run = False
                break

            with self._lock:
                # Find primary spinner (first registered).
                primary_spinner = next((s for s in self._spinners if s._is_primary), None)

                if primary_spinner and primary_spinner.running:
                    # Render in the same line.
                    render_line = primary_spinner._generate_render_line()

                    # Append additional spinner info if multiple exist.
                    if len(self._spinners) > 1:
                        secondary_info = " | ".join(
                            s._generate_secondary_info()
                            for s in self._spinners
                            if s is not primary_spinner and s.running
                        )
                        render_line += f" [{secondary_info}]"

                    # Clear line and write.
                    try:
                        sys.stdout.write("\r" + render_line + "\033[K")
                        sys.stdout.flush()
                    except Exception:
                        self._should_run = False

            time.sleep(0.1)  # Render interval.
register_spinner(spinner)

Register a new spinner.

Source code in toolboxv2/utils/extras/Style.py
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
def register_spinner(self, spinner):
    """Register a new spinner."""
    with self._lock:
        # First spinner defines the rendering line.
        if not self._spinners:
            spinner._is_primary = True
        self._spinners.append(spinner)
        # Start rendering if not already running.
        if not self._should_run:
            self._should_run = True
            self._render_thread = threading.Thread(
                target=self._render_loop,
                daemon=True
            )
            self._render_thread.start()
unregister_spinner(spinner)

Unregister a completed spinner.

Source code in toolboxv2/utils/extras/Style.py
539
540
541
542
543
def unregister_spinner(self, spinner):
    """Unregister a completed spinner."""
    with self._lock:
        if spinner in self._spinners:
            self._spinners.remove(spinner)
base_widget
BaseWidget
Source code in toolboxv2/utils/extras/base_widget.py
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
class BaseWidget:
    def __init__(self, name: str):
        self.name = name
        self.openWidgetsIDs = {}
        self.onReload = []
        self.iframes = {}

    def register(self, app, fuction, version=None, name="get_widget", level=1, **kwargs):
        if version is None:
            version = app.version
        app.tb(mod_name=self.name, version=version, request_as_kwarg=True, level=level, api=True, name=name, **kwargs)(
            fuction)

    def modify_iterator(self, iterator, replace):
        """
        ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
        {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
        """

        for item in iterator:
            modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                             range(len(replace))}
            yield modified_item

    def register2reload(self, *functions):
        for fuction in functions:
            def x(r):
                return fuction(request=r)
            self.onReload.append(x)

    def reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = function()
        return c

    async def oa_reload_guard(self, function):
        c = None
        if len(self.onReload) == 0:
            c = await function() if asyncio.iscoroutinefunction(function) else function()
        return c

    @staticmethod
    def get_a_group(asset_name, template=None, file_path=None, a_kwargs=None):
        if a_kwargs is None:
            raise ValueError("a_kwargs must be specified")
        return [{'name': asset_name,
                 'file_path': file_path,
                 'kwargs': a_kwargs
                 } if file_path is not None else {'name': asset_name,
                                                  'template': template,
                                                  'kwargs': a_kwargs
                                                  }]

    def group_generator(self, asset_name: str, iterator: iter, template=None, file_path=None, a_kwargs=None):
        groups = []
        work_kwargs = a_kwargs
        for _i, data in enumerate(iterator):
            if isinstance(data, dict):
                work_kwargs = {**a_kwargs, **data}
            groups.append(self.get_a_group(asset_name, template=template, file_path=file_path, a_kwargs=work_kwargs))
        return groups

    def asset_loder(self, app, name, asset_id, file_path=None, template=None, iterator=None, **kwargs):
        a_kwargs = {**{
            'root': f"/api/{self.name}",
            'WidgetID': asset_id},
                    **kwargs}
        asset_name = f"{name}-{asset_id}"
        if iterator is None:
            group = self.get_a_group(asset_name,
                                     template=template,
                                     file_path=file_path,
                                     a_kwargs=a_kwargs)
        else:
            group = self.group_generator(asset_name,
                                         iterator=iterator,
                                         template=template,
                                         file_path=file_path,
                                         a_kwargs=a_kwargs)

        asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                            group_name=self.name,
                            collection={'name': f"{asset_name}",
                                        'group': group},
                            get_results=True)
        if asset.is_error():
            app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
            asset = app.run_any(MINIMALHTML.ADD_COLLECTION_TO_GROUP,
                                group_name=self.name,
                                collection={'name': f"{self.name}-{asset_name}",
                                            'group': group},
                                get_results=True)
        return asset

    def generate_html(self, app, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        return app.run_any(MINIMALHTML.GENERATE_HTML,
                           group_name=self.name,
                           collection_name=f"{name}-{asset_id}")

    def load_widget(self, app, request, name="MainWidget", asset_id=str(uuid.uuid4())[:4]):
        app.run_any(MINIMALHTML.ADD_GROUP, command=self.name)
        self.reload(request)
        html_widget = self.generate_html(app, name, asset_id)
        return html_widget[0]['html_element']

    @staticmethod
    async def get_user_from_request(app, request):
        from toolboxv2.mods.CloudM import User
        if request is None:
            return User()
        return await get_current_user_from_request(app, request)

    @staticmethod
    def get_s_id(request):
        from ..system.types import Result
        if request is None:
            return Result.default_internal_error("No request specified")
        return Result.ok(request.session.get('ID', ''))

    def reload(self, request):
        [_(request) for _ in self.onReload]

    async def oa_reload(self, request):
        [_(request) if not asyncio.iscoroutinefunction(_) else await _(request) for _ in self.onReload]

    async def get_widget(self, request, **kwargs):
        raise NotImplementedError

    def hash_wrapper(self, _id, _salt=''):
        from ..security.cryp import Code
        return Code.one_way_hash(text=_id, salt=_salt, pepper=self.name)

    def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
        """
        Registriert einen iframe mit gegebener ID und Quelle

        Args:
            iframe_id: Eindeutige ID für den iframe
            src: URL oder Pfad zur Quelle des iframes
            width: Breite des iframes (default: "100%")
            height: Höhe des iframes (default: "500px")
            **kwargs: Weitere iframe-Attribute
        """
        iframe_config = {
            'src': src,
            'width': width,
            'height': height,
            **kwargs
        }
        self.iframes[iframe_id] = iframe_config

    def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
        """
        Erstellt ein Asset für einen registrierten iframe

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        if iframe_id not in self.iframes:
            raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

        if asset_id is None:
            asset_id = str(uuid.uuid4())[:4]

        iframe_config = self.iframes[iframe_id]
        iframe_template = """
        <iframe id="{iframe_id}"
                src="{src}"
                width="{width}"
                height="{height}"
                frameborder="0"
                {additional_attrs}></iframe>
        """.strip()

        # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
        known_attrs = {'src', 'width', 'height'}
        additional_attrs = ' '.join(
            f'{k}="{v}"' for k, v in iframe_config.items()
            if k not in known_attrs
        )

        iframe_html = iframe_template.format(
            iframe_id=iframe_id,
            src=iframe_config['src'],
            width=iframe_config['width'],
            height=iframe_config['height'],
            additional_attrs=additional_attrs
        )

        return self.asset_loder(
            app=app,
            name=f"iframe-{iframe_id}",
            asset_id=asset_id,
            template=iframe_html
        )

    def load_iframe(self, app, iframe_id: str, asset_id: str = None):
        """
        Lädt einen registrierten iframe und gibt das HTML-Element zurück

        Args:
            app: App-Instanz
            iframe_id: ID des registrierten iframes
            asset_id: Optional, spezifische Asset-ID
        """
        self.create_iframe_asset(app, iframe_id, asset_id)
        return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']
create_iframe_asset(app, iframe_id, asset_id=None)

Erstellt ein Asset für einen registrierten iframe

Parameters:

Name Type Description Default
app

App-Instanz

required
iframe_id str

ID des registrierten iframes

required
asset_id str

Optional, spezifische Asset-ID

None
Source code in toolboxv2/utils/extras/base_widget.py
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
def create_iframe_asset(self, app, iframe_id: str, asset_id: str = None):
    """
    Erstellt ein Asset für einen registrierten iframe

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    if iframe_id not in self.iframes:
        raise ValueError(f"iframe mit ID {iframe_id} nicht registriert")

    if asset_id is None:
        asset_id = str(uuid.uuid4())[:4]

    iframe_config = self.iframes[iframe_id]
    iframe_template = """
    <iframe id="{iframe_id}"
            src="{src}"
            width="{width}"
            height="{height}"
            frameborder="0"
            {additional_attrs}></iframe>
    """.strip()

    # Filtere bekannte Attribute heraus und erstelle String für zusätzliche Attribute
    known_attrs = {'src', 'width', 'height'}
    additional_attrs = ' '.join(
        f'{k}="{v}"' for k, v in iframe_config.items()
        if k not in known_attrs
    )

    iframe_html = iframe_template.format(
        iframe_id=iframe_id,
        src=iframe_config['src'],
        width=iframe_config['width'],
        height=iframe_config['height'],
        additional_attrs=additional_attrs
    )

    return self.asset_loder(
        app=app,
        name=f"iframe-{iframe_id}",
        asset_id=asset_id,
        template=iframe_html
    )
load_iframe(app, iframe_id, asset_id=None)

Lädt einen registrierten iframe und gibt das HTML-Element zurück

Parameters:

Name Type Description Default
app

App-Instanz

required
iframe_id str

ID des registrierten iframes

required
asset_id str

Optional, spezifische Asset-ID

None
Source code in toolboxv2/utils/extras/base_widget.py
280
281
282
283
284
285
286
287
288
289
290
def load_iframe(self, app, iframe_id: str, asset_id: str = None):
    """
    Lädt einen registrierten iframe und gibt das HTML-Element zurück

    Args:
        app: App-Instanz
        iframe_id: ID des registrierten iframes
        asset_id: Optional, spezifische Asset-ID
    """
    self.create_iframe_asset(app, iframe_id, asset_id)
    return self.generate_html(app, f"iframe-{iframe_id}", asset_id)[0]['html_element']
modify_iterator(iterator, replace)

['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'}, {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]

Source code in toolboxv2/utils/extras/base_widget.py
 94
 95
 96
 97
 98
 99
100
101
102
103
def modify_iterator(self, iterator, replace):
    """
    ['a', 'b'] -> [{replace[0]: 'a',..., replace[len(replace)-1]: 'a'},
    {replace[0]: 'b',..., replace[len(replace)-1]: 'b'}, ]
    """

    for item in iterator:
        modified_item = {replace[i]: (self.name if replace[i] == "name" else '') + item for i in
                         range(len(replace))}
        yield modified_item
register_iframe(iframe_id, src, width='100%', height='500px', **kwargs)

Registriert einen iframe mit gegebener ID und Quelle

Parameters:

Name Type Description Default
iframe_id str

Eindeutige ID für den iframe

required
src str

URL oder Pfad zur Quelle des iframes

required
width str

Breite des iframes (default: "100%")

'100%'
height str

Höhe des iframes (default: "500px")

'500px'
**kwargs

Weitere iframe-Attribute

{}
Source code in toolboxv2/utils/extras/base_widget.py
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def register_iframe(self, iframe_id: str, src: str, width: str = "100%", height: str = "500px", **kwargs):
    """
    Registriert einen iframe mit gegebener ID und Quelle

    Args:
        iframe_id: Eindeutige ID für den iframe
        src: URL oder Pfad zur Quelle des iframes
        width: Breite des iframes (default: "100%")
        height: Höhe des iframes (default: "500px")
        **kwargs: Weitere iframe-Attribute
    """
    iframe_config = {
        'src': src,
        'width': width,
        'height': height,
        **kwargs
    }
    self.iframes[iframe_id] = iframe_config
blobs
BlobFile
Source code in toolboxv2/utils/extras/blobs.py
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
class BlobFile(io.IOBase):
    def __init__(self, filename: str, mode: str = 'r', storage: BlobStorage = None, key: str = None,
                 servers: list[str] = None):
        if not isinstance(filename, str) or not filename:
            raise ValueError("Filename must be a non-empty string.")
        if not filename.startswith('/'): filename = '/' + filename
        self.filename = filename.lstrip('/\\')
        self.blob_id, self.folder, self.datei = self._path_splitter(self.filename)
        self.mode = mode

        if storage is None:
            # In a real app, dependency injection or a global factory would be better
            # but this provides a fallback for simple scripts.
            if not servers:
                from toolboxv2 import get_app
                storage = get_app(from_="BlobStorage").root_blob_storage
            else:
                storage = BlobStorage(servers=servers)

        self.storage = storage
        self.data_buffer = b""
        self.key = key
        if key:
            try:
                assert Code.decrypt_symmetric(Code.encrypt_symmetric(b"test", key), key, to_str=False) == b"test"
            except Exception:
                raise ValueError("Invalid symmetric key provided.")

    @staticmethod
    def _path_splitter(filename):
        parts = Path(filename).parts
        if not parts: raise ValueError("Filename cannot be empty.")
        blob_id = parts[0]
        if len(parts) == 1: raise ValueError("Filename must include a path within the blob, e.g., 'blob_id/file.txt'")
        datei = parts[-1]
        folder = '|'.join(parts[1:-1])
        return blob_id, folder, datei

    def create(self):
        self.storage.create_blob(pickle.dumps({}), self.blob_id)
        return self

    def __enter__(self):
        try:
            raw_blob_data = self.storage.read_blob(self.blob_id)
            if raw_blob_data != b'' and (not raw_blob_data or raw_blob_data is None):
                raw_blob_data = b""
            blob_content = pickle.loads(raw_blob_data)
        except (requests.exceptions.HTTPError, EOFError, pickle.UnpicklingError, ConnectionError) as e:
            if isinstance(e, requests.exceptions.HTTPError) and e.response.status_code == 404:
                blob_content = {}  # Blob doesn't exist yet, treat as empty
            elif isinstance(e, EOFError | pickle.UnpicklingError):
                blob_content = {}  # Blob is empty or corrupt, treat as empty for writing
            else:
                self.storage.create_blob(blob_id=self.blob_id, data=pickle.dumps({}))
                blob_content = {}

        if 'r' in self.mode:
            path_key = self.folder if self.folder else self.datei
            if self.folder:
                file_data = blob_content.get(self.folder, {}).get(self.datei)
            else:
                file_data = blob_content.get(self.datei)

            if file_data:
                self.data_buffer = file_data
                if self.key:
                    self.data_buffer = Code.decrypt_symmetric(self.data_buffer, self.key, to_str=False)
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if 'w' in self.mode:
            final_data = self.data_buffer
            if self.key:
                final_data = Code.encrypt_symmetric(final_data, self.key)

            try:
                raw_blob_data = self.storage.read_blob(self.blob_id)
                blob_content = pickle.loads(raw_blob_data)
            except Exception:
                blob_content = {}

            # Safely navigate and create path
            current_level = blob_content
            if self.folder:
                if self.folder not in current_level:
                    current_level[self.folder] = {}
                current_level = current_level[self.folder]

            current_level[self.datei] = final_data
            self.storage.update_blob(self.blob_id, pickle.dumps(blob_content))




    def exists(self) -> bool:
        """
        Checks if the specific file path exists within the blob without reading its content.
        This is an efficient, read-only operation.

        Returns:
            bool: True if the file exists within the blob, False otherwise.
        """
        try:
            # Fetch the raw blob data. This leverages the local cache if available.
            raw_blob_data = self.storage.read_blob(self.blob_id)
            # Unpickle the directory structure.
            if raw_blob_data:
                blob_content = pickle.loads(raw_blob_data)
            else:
                return False
        except (requests.exceptions.HTTPError, EOFError, pickle.UnpicklingError, ConnectionError):
            # If the blob itself doesn't exist, is empty, or can't be reached,
            # then the file within it cannot exist.
            return False

        # Navigate the dictionary to check for the file's existence.
        current_level = blob_content
        if self.folder:
            if self.folder not in current_level:
                return False
            current_level = current_level[self.folder]

        return self.datei in current_level

    def clear(self):
        self.data_buffer = b''

    def write(self, data):
        if 'w' not in self.mode: raise OSError("File not opened in write mode.")
        if isinstance(data, str):
            self.data_buffer += data.encode()
        elif isinstance(data, bytes):
            self.data_buffer += data
        else:
            raise TypeError("write() argument must be str or bytes")

    def read(self):
        if 'r' not in self.mode: raise OSError("File not opened in read mode.")
        return self.data_buffer

    def read_json(self):
        if 'r' not in self.mode: raise ValueError("File not opened in read mode.")
        if self.data_buffer == b"": return {}
        return json.loads(self.data_buffer.decode())

    def write_json(self, data):
        if 'w' not in self.mode: raise ValueError("File not opened in write mode.")
        self.data_buffer += json.dumps(data).encode()

    def read_pickle(self):
        if 'r' not in self.mode: raise ValueError("File not opened in read mode.")
        if self.data_buffer == b"": return {}
        return pickle.loads(self.data_buffer)

    def write_pickle(self, data):
        if 'w' not in self.mode: raise ValueError("File not opened in write mode.")
        self.data_buffer += pickle.dumps(data)

    def read_yaml(self):
        if 'r' not in self.mode: raise ValueError("File not opened in read mode.")
        if self.data_buffer == b"": return {}
        return yaml.safe_load(self.data_buffer)

    def write_yaml(self, data):
        if 'w' not in self.mode: raise ValueError("File not opened in write mode.")
        yaml.dump(data, self)
exists()

Checks if the specific file path exists within the blob without reading its content. This is an efficient, read-only operation.

Returns:

Name Type Description
bool bool

True if the file exists within the blob, False otherwise.

Source code in toolboxv2/utils/extras/blobs.py
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
def exists(self) -> bool:
    """
    Checks if the specific file path exists within the blob without reading its content.
    This is an efficient, read-only operation.

    Returns:
        bool: True if the file exists within the blob, False otherwise.
    """
    try:
        # Fetch the raw blob data. This leverages the local cache if available.
        raw_blob_data = self.storage.read_blob(self.blob_id)
        # Unpickle the directory structure.
        if raw_blob_data:
            blob_content = pickle.loads(raw_blob_data)
        else:
            return False
    except (requests.exceptions.HTTPError, EOFError, pickle.UnpicklingError, ConnectionError):
        # If the blob itself doesn't exist, is empty, or can't be reached,
        # then the file within it cannot exist.
        return False

    # Navigate the dictionary to check for the file's existence.
    current_level = blob_content
    if self.folder:
        if self.folder not in current_level:
            return False
        current_level = current_level[self.folder]

    return self.datei in current_level
BlobStorage

A production-ready client for the distributed blob storage server. It handles communication with a list of server instances, manages a local cache, and implements backoff/retry logic for resilience.

Source code in toolboxv2/utils/extras/blobs.py
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
class BlobStorage:
    """
    A production-ready client for the distributed blob storage server.
    It handles communication with a list of server instances, manages a local cache,
    and implements backoff/retry logic for resilience.
    """

    def __init__(self, servers: list[str], storage_directory: str = './.data/blob_cache'):


        self.servers = servers
        self.session = requests.Session()
        self.storage_directory = storage_directory
        self.blob_ids = []
        os.makedirs(storage_directory, exist_ok=True)

        # Initialize the consistent hash ring
        self.hash_ring = ConsistentHashRing()
        for server in self.servers:
            self.hash_ring.add_node(server)

    def _make_request(self, method, endpoint, blob_id: str = None, max_retries=2, **kwargs):
        """
        Makes a resilient HTTP request to the server cluster.
        - If a blob_id is provided, it uses the consistent hash ring to find the
          primary server and subsequent backup servers in a predictable order.
        - If no blob_id is given (e.g., for broadcast actions), it tries servers randomly.
        - Implements exponential backoff on server errors.
        """
        if not self.servers:
            res = requests.Response()
            res.status_code = 503
            res.reason = "No servers available"
            return res

        if blob_id:
            # Get the ordered list of servers for this specific blob
            preferred_servers = self.hash_ring.get_nodes_for_key(blob_id)
        else:
            # For non-specific requests, shuffle all servers
            preferred_servers = random.sample(self.servers, len(self.servers))

        last_error = None
        for attempt in range(max_retries):
            for server in preferred_servers:
                url = f"{server.rstrip('/')}{endpoint}"
                try:
                    # In a targeted request, print which server we are trying
                    response = self.session.request(method, url, timeout=10, **kwargs)

                    if 500 <= response.status_code < 600:
                        get_logger().warning(f"Warning: Server {server} returned status {response.status_code}. Retrying...")
                        continue
                    response.raise_for_status()
                    return response
                except requests.exceptions.RequestException as e:
                    last_error = e
                    get_logger().warning(f"Warning: Could not connect to server {server}: {e}. Trying next server.")

            if attempt < max_retries - 1:
                wait_time = 2 ** (attempt*0.1)
                get_logger().warning(f"Warning: All preferred servers failed. Retrying in {wait_time} seconds...")
                time.sleep(wait_time)
                if len(preferred_servers) == 1 and len(self.servers) > 1:
                    preferred_servers = random.sample(self.servers, len(self.servers))

        raise ConnectionError(f"Failed to execute request after {max_retries} attempts. Last error: {last_error}")


    def create_blob(self, data: bytes, blob_id=None) -> str:
        """
        Creates a new blob. The blob_id is calculated client-side by hashing
        the content, and the data is sent to the correct server determined
        by the consistent hash ring. This uses a PUT request, making creation
        idempotent.
        """
        # The blob ID is the hash of its content, ensuring content-addressable storage.
        if not blob_id:
            blob_id = hashlib.sha256(data).hexdigest()

        # Use PUT, as we now know the blob's final ID/URL.
        # Pass blob_id to _make_request so it uses the hash ring.
        print(f"Creating blob {blob_id} on {self._make_request('PUT', f'/blob/{blob_id}',blob_id=blob_id, data=data).status_code}")
        # blob_id = response.text
        self._save_blob_to_cache(blob_id, data)
        return blob_id

    def read_blob(self, blob_id: str) -> bytes:
        cached_data = self._load_blob_from_cache(blob_id)
        if cached_data is not None:
            return cached_data

        get_logger().info(f"Info: Blob '{blob_id}' not in cache, fetching from network.")
        # Pass blob_id to _make_request to target the correct server(s).
        response = self._make_request('GET', f'/blob/{blob_id}', blob_id=blob_id)

        blob_data = response.content
        self._save_blob_to_cache(blob_id, blob_data)
        return blob_data

    def update_blob(self, blob_id: str, data: bytes):
        # Pass blob_id to _make_request to target the correct server(s).
        response = self._make_request('PUT', f'/blob/{blob_id}', blob_id=blob_id, data=data)
        self._save_blob_to_cache(blob_id, data)
        return response

    def delete_blob(self, blob_id: str):
        # Pass blob_id to _make_request to target the correct server(s).
        self._make_request('DELETE', f'/blob/{blob_id}', blob_id=blob_id)
        cache_file = self._get_blob_cache_filename(blob_id)
        if os.path.exists(cache_file):
            os.remove(cache_file)

    # NOTE: share_blobs and recover_blob are coordination endpoints. They do not
    # act on a single blob, so they will continue to use the non-targeted (random)
    # request mode to contact any available server to act as a coordinator.
    def share_blobs(self, blob_ids: list[str]):
        get_logger().info(f"Info: Instructing a server to share blobs for recovery: {blob_ids}")
        payload = {"blob_ids": blob_ids}
        # No blob_id passed, will try any server as a coordinator.
        self._make_request('POST', '/share', json=payload)
        get_logger().info("Info: Sharing command sent successfully.")

    def recover_blob(self, lost_blob_id: str) -> bytes:
        get_logger().info(f"Info: Attempting to recover '{lost_blob_id}' from the cluster.")
        payload = {"blob_id": lost_blob_id}
        # No blob_id passed, recovery can be initiated by any server.
        response = self._make_request('POST', '/recover', json=payload)

        recovered_data = response.content
        get_logger().info(f"Info: Successfully recovered blob '{lost_blob_id}'.")
        self._save_blob_to_cache(lost_blob_id, recovered_data)
        return recovered_data

    def _get_blob_cache_filename(self, blob_id: str) -> str:
        return os.path.join(self.storage_directory, blob_id + '.blobcache')

    def _save_blob_to_cache(self, blob_id: str, data: bytes):
        if not data or data is None:
            return
        if blob_id not in self.blob_ids:
            self.blob_ids.append(blob_id)
        with open(self._get_blob_cache_filename(blob_id), 'wb') as f:
            f.write(data)

    def _load_blob_from_cache(self, blob_id: str) -> bytes | None:
        cache_file = self._get_blob_cache_filename(blob_id)
        if not os.path.exists(cache_file):
            return None
        with open(cache_file, 'rb') as f:
            return f.read()

    def exit(self):
        if len(self.blob_ids) < 5:
            return
        for _i in range(len(self.servers)//2+1):
            self.share_blobs(self.blob_ids)
create_blob(data, blob_id=None)

Creates a new blob. The blob_id is calculated client-side by hashing the content, and the data is sent to the correct server determined by the consistent hash ring. This uses a PUT request, making creation idempotent.

Source code in toolboxv2/utils/extras/blobs.py
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
def create_blob(self, data: bytes, blob_id=None) -> str:
    """
    Creates a new blob. The blob_id is calculated client-side by hashing
    the content, and the data is sent to the correct server determined
    by the consistent hash ring. This uses a PUT request, making creation
    idempotent.
    """
    # The blob ID is the hash of its content, ensuring content-addressable storage.
    if not blob_id:
        blob_id = hashlib.sha256(data).hexdigest()

    # Use PUT, as we now know the blob's final ID/URL.
    # Pass blob_id to _make_request so it uses the hash ring.
    print(f"Creating blob {blob_id} on {self._make_request('PUT', f'/blob/{blob_id}',blob_id=blob_id, data=data).status_code}")
    # blob_id = response.text
    self._save_blob_to_cache(blob_id, data)
    return blob_id
ConsistentHashRing

A consistent hash ring implementation to map keys (blob_ids) to nodes (servers). It uses virtual nodes (replicas) to ensure a more uniform distribution of keys.

Source code in toolboxv2/utils/extras/blobs.py
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
class ConsistentHashRing:
    """
    A consistent hash ring implementation to map keys (blob_ids) to nodes (servers).
    It uses virtual nodes (replicas) to ensure a more uniform distribution of keys.
    """
    def __init__(self, replicas=100):
        """
        :param replicas: The number of virtual nodes for each physical node.
                         Higher values lead to more balanced distribution.
        """
        self.replicas = replicas
        self._keys = []  # Sorted list of hash values (the ring)
        self._nodes = {} # Maps hash values to physical node URLs

    def _hash(self, key: str) -> int:
        """Hashes a key to an integer using md5 for speed and distribution."""
        return int(hashlib.md5(key.encode('utf-8')).hexdigest(), 16)

    def add_node(self, node: str):
        """Adds a physical node to the hash ring."""
        for i in range(self.replicas):
            vnode_key = f"{node}:{i}"
            h = self._hash(vnode_key)
            bisect.insort(self._keys, h)
            self._nodes[h] = node

    def get_nodes_for_key(self, key: str) -> list[str]:
        """
        Returns an ordered list of nodes responsible for the given key.
        The first node in the list is the primary, the rest are failover candidates
        in preferential order.
        """
        if not self._nodes:
            return []

        h = self._hash(key)
        start_idx = bisect.bisect_left(self._keys, h)

        # Collect unique physical nodes by iterating around the ring
        found_nodes = []
        for i in range(len(self._keys)):
            idx = (start_idx + i) % len(self._keys)
            node_hash = self._keys[idx]
            physical_node = self._nodes[node_hash]
            if physical_node not in found_nodes:
                found_nodes.append(physical_node)
            # Stop when we have found all unique physical nodes
            if len(found_nodes) == len(set(self._nodes.values())):
                break
        return found_nodes
__init__(replicas=100)

:param replicas: The number of virtual nodes for each physical node. Higher values lead to more balanced distribution.

Source code in toolboxv2/utils/extras/blobs.py
25
26
27
28
29
30
31
32
def __init__(self, replicas=100):
    """
    :param replicas: The number of virtual nodes for each physical node.
                     Higher values lead to more balanced distribution.
    """
    self.replicas = replicas
    self._keys = []  # Sorted list of hash values (the ring)
    self._nodes = {} # Maps hash values to physical node URLs
add_node(node)

Adds a physical node to the hash ring.

Source code in toolboxv2/utils/extras/blobs.py
38
39
40
41
42
43
44
def add_node(self, node: str):
    """Adds a physical node to the hash ring."""
    for i in range(self.replicas):
        vnode_key = f"{node}:{i}"
        h = self._hash(vnode_key)
        bisect.insort(self._keys, h)
        self._nodes[h] = node
get_nodes_for_key(key)

Returns an ordered list of nodes responsible for the given key. The first node in the list is the primary, the rest are failover candidates in preferential order.

Source code in toolboxv2/utils/extras/blobs.py
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
def get_nodes_for_key(self, key: str) -> list[str]:
    """
    Returns an ordered list of nodes responsible for the given key.
    The first node in the list is the primary, the rest are failover candidates
    in preferential order.
    """
    if not self._nodes:
        return []

    h = self._hash(key)
    start_idx = bisect.bisect_left(self._keys, h)

    # Collect unique physical nodes by iterating around the ring
    found_nodes = []
    for i in range(len(self._keys)):
        idx = (start_idx + i) % len(self._keys)
        node_hash = self._keys[idx]
        physical_node = self._nodes[node_hash]
        if physical_node not in found_nodes:
            found_nodes.append(physical_node)
        # Stop when we have found all unique physical nodes
        if len(found_nodes) == len(set(self._nodes.values())):
            break
    return found_nodes
gist_control
GistLoader
Source code in toolboxv2/utils/extras/gist_control.py
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class GistLoader:
    def __init__(self, gist_url):
        self.gist_url = gist_url
        self.module_code = None

    def load_module(self, module_name):
        """Lädt das Modul mit dem gegebenen Namen."""
        if self.module_code is None:
            self.module_code = self._fetch_gist_content()

        # Erstelle ein neues Modul
        module = importlib.util.module_from_spec(self.get_spec(module_name))
        exec(self.module_code, module.__dict__)
        return module

    def get_spec(self, module_name):
        """Gibt die Modul-Specifikation zurück."""
        return ModuleSpec(module_name, self)

    def get_filename(self, module_name):
        return f"<gist:{self.gist_url}>"

    def _fetch_gist_content(self):
        """Lädt den Inhalt des Gists von der GitHub API herunter."""
        gist_id = self.gist_url.split('/')[-1]
        api_url = f"https://api.github.com/gists/{gist_id}"

        response = requests.get(api_url)

        if response.status_code == 200:
            gist_data = response.json()
            first_file = next(iter(gist_data['files'].values()))
            return first_file['content']
        else:
            raise Exception(f"Failed to fetch gist: {response.status_code}")
get_spec(module_name)

Gibt die Modul-Specifikation zurück.

Source code in toolboxv2/utils/extras/gist_control.py
23
24
25
def get_spec(self, module_name):
    """Gibt die Modul-Specifikation zurück."""
    return ModuleSpec(module_name, self)
load_module(module_name)

Lädt das Modul mit dem gegebenen Namen.

Source code in toolboxv2/utils/extras/gist_control.py
13
14
15
16
17
18
19
20
21
def load_module(self, module_name):
    """Lädt das Modul mit dem gegebenen Namen."""
    if self.module_code is None:
        self.module_code = self._fetch_gist_content()

    # Erstelle ein neues Modul
    module = importlib.util.module_from_spec(self.get_spec(module_name))
    exec(self.module_code, module.__dict__)
    return module
helper_test_functions
generate_edge_value(param_type)

Generiert Edge-Case-Werte basierend auf dem Parametertyp.

Source code in toolboxv2/utils/extras/helper_test_functions.py
35
36
37
38
39
40
41
42
43
44
def generate_edge_value(param_type: Any) -> Any:
    """
    Generiert Edge-Case-Werte basierend auf dem Parametertyp.
    """
    if param_type in [int, float]:
        return -999  # Beispiel für negative Zahlen
    elif param_type == str:
        return "test " * 100  # Lange zufällige Strings
    # Fügen Sie hier weitere Bedingungen für andere Datentypen hinzu
    return None
generate_normal_value(param_type)

Generiert normale Werte basierend auf dem Parametertyp.

Source code in toolboxv2/utils/extras/helper_test_functions.py
47
48
49
50
51
52
53
54
55
56
57
58
59
def generate_normal_value(param_type: Any) -> Any:
    """
    Generiert normale Werte basierend auf dem Parametertyp.
    """
    from toolboxv2 import RequestData
    if param_type in [int, float]:
        return random.randint(0, 100)  # Zufällige normale Zahlen
    elif param_type == str:
        return "test" # Zufälliges Wort
    elif param_type == RequestData:
        return RequestData.moc()
    # Fügen Sie hier weitere Bedingungen für andere Datentypen hinzu
    return None
keword_matcher
calculate_keyword_score(text, keywords)

Berechnet den Keyword-Score basierend auf der Häufigkeit der Keywords im Text. Case-insensitive und optimiert für Geschwindigkeit.

:param text: Eingabetext als String :param keywords: Set von Keywords :return: Gesamt-Score als Integer

Source code in toolboxv2/utils/extras/keword_matcher.py
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
def calculate_keyword_score(text: str, keywords: set[str]) -> int:
    """
    Berechnet den Keyword-Score basierend auf der Häufigkeit der Keywords im Text.
    Case-insensitive und optimiert für Geschwindigkeit.

    :param text: Eingabetext als String
    :param keywords: Set von Keywords
    :return: Gesamt-Score als Integer
    """
    # Vorverarbeitung der Keywords
    keyword_pattern = re.compile(
        r'\b(' + '|'.join(re.escape(k.lower()) for k in keywords) + r')\b',
        flags=re.IGNORECASE
    )

    # Erstelle Frequenz-Wörterbuch
    freq_dict = defaultdict(int)

    # Finde alle Übereinstimmungen
    matches = keyword_pattern.findall(text.lower())

    # Zähle die Treffer
    for match in matches:
        freq_dict[match.lower()] += 1

    # Berechne Gesamt-Score
    total_score = sum(freq_dict.values())

    return total_score
calculate_weighted_score(text, keyword_weights)

Berechnet gewichteten Score mit unterschiedlichen Gewichten pro Keyword

:param text: Eingabetext :param keyword_weights: Dictionary mit {Keyword: Gewicht} :return: Gewichteter Gesamt-Score

Source code in toolboxv2/utils/extras/keword_matcher.py
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
def calculate_weighted_score(text: str, keyword_weights: dict or list) -> float:
    """
    Berechnet gewichteten Score mit unterschiedlichen Gewichten pro Keyword

    :param text: Eingabetext
    :param keyword_weights: Dictionary mit {Keyword: Gewicht}
    :return: Gewichteter Gesamt-Score
    """
    total = 0.0
    text_lower = text.lower()

    if isinstance(keyword_weights, list):
        keyword_weights = {k:v for k, v in keyword_weights}

    for keyword, weight in keyword_weights.items():
        count = len(re.findall(r'\b' + re.escape(keyword.lower()) + r'\b', text_lower))
        total += count * weight

    return round(total, 2)
extract_keywords(text, max_len=-1, min_word_length=3, with_weights=False, remove_stopwords=True, stopwords=True)

Extrahiert Keywords mit optionaler Frequenzgewichtung

:param text: Eingabetext :param max_len: Maximale Anzahl Keywords (-1 = alle) :param min_word_length: Minimale Wortlänge :param with_weights: Gibt Wort+Frequenz zurück wenn True :param remove_stopwords: Filtert deutsche Stopwörter :param german_stopwords: Verwendet deutsche Standard-Stopwörter :return: Keywords oder (Keyword, Häufigkeit) Paare

Source code in toolboxv2/utils/extras/keword_matcher.py
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
def extract_keywords(
    text: str,
    max_len: int = -1,
    min_word_length: int = 3,
    with_weights: bool = False,
    remove_stopwords: bool = True,
    stopwords: bool = True
) -> list[str] | list[tuple[str, int]]:
    """
    Extrahiert Keywords mit optionaler Frequenzgewichtung

    :param text: Eingabetext
    :param max_len: Maximale Anzahl Keywords (-1 = alle)
    :param min_word_length: Minimale Wortlänge
    :param with_weights: Gibt Wort+Frequenz zurück wenn True
    :param remove_stopwords: Filtert deutsche Stopwörter
    :param german_stopwords: Verwendet deutsche Standard-Stopwörter
    :return: Keywords oder (Keyword, Häufigkeit) Paare
    """

    # Deutsche Basis-Stopwörter
    DEFAULT_STOPWORDS = STOPWORDS if stopwords else set()

    # Text vorverarbeiten
    words = re.findall(r'\b\w+\b', text.lower())

    # Worte filtern
    filtered_words = [
        word for word in words
        if len(word) > min_word_length
           and (not remove_stopwords or word not in DEFAULT_STOPWORDS)
    ]

    # Frequenzanalyse
    word_counts = defaultdict(int)
    for word in filtered_words:
        word_counts[word] += 1

    # Sortierung: Zuerst Häufigkeit, dann alphabetisch
    sorted_words = sorted(
        word_counts.items(),
        key=lambda x: (-x[1], x[0])
    )

    # Längenbegrenzung
    if max_len == -1:
        max_len = None
    result = sorted_words[:max_len]

    return result if with_weights else [word for word, _ in result]
mkdocs
CodeElement dataclass

Represents a code element (class, function, etc.)

Source code in toolboxv2/utils/extras/mkdocs.py
48
49
50
51
52
53
54
55
56
57
58
59
@dataclass
class CodeElement:
    """Represents a code element (class, function, etc.)"""
    name: str
    element_type: str
    file_path: str
    line_start: int
    line_end: int
    signature: str
    docstring: Optional[str] = None
    hash_signature: str = ""
    parent_class: Optional[str] = None
CodeElementExtractor

Extracts code elements from source files

Source code in toolboxv2/utils/extras/mkdocs.py
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
class CodeElementExtractor:
    """Extracts code elements from source files"""

    def extract_python_elements(self, file_path: Path) -> List[CodeElement]:
        """Extract elements from Python file"""
        elements = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            tree = ast.parse(content)

            for node in ast.walk(tree):
                if isinstance(node, ast.ClassDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='class',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=f"class {node.name}",
                        docstring=ast.get_docstring(node),
                        hash_signature=self._hash_node(node, content)
                    ))

                    # Extract methods
                    for item in node.body:
                        if isinstance(item, ast.FunctionDef):
                            elements.append(CodeElement(
                                name=item.name,
                                element_type='method',
                                file_path=str(file_path),
                                line_start=item.lineno,
                                line_end=getattr(item, 'end_lineno', item.lineno),
                                signature=self._get_function_signature(item),
                                docstring=ast.get_docstring(item),
                                parent_class=node.name,
                                hash_signature=self._hash_node(item, content)
                            ))

                elif isinstance(node, ast.FunctionDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='function',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=self._get_function_signature(node),
                        docstring=ast.get_docstring(node),
                        hash_signature=self._hash_node(node, content)
                    ))

        except Exception as e:
            logger.error(f"Error extracting Python elements from {file_path}: {e}")

        return elements

    def _get_function_signature(self, node: ast.FunctionDef) -> str:
        """Get function signature string"""
        args = []
        for arg in node.args.args:
            args.append(arg.arg)

        return f"def {node.name}({', '.join(args)})"

    def _hash_node(self, node: ast.AST, content: str) -> str:
        """Generate hash for AST node"""
        try:
            node_source = ast.unparse(node)
            return hashlib.md5(node_source.encode()).hexdigest()
        except:
            # Fallback: use line-based hash
            lines = content.split('\n')
            start = getattr(node, 'lineno', 1) - 1
            end = getattr(node, 'end_lineno', start + 1)
            node_content = '\n'.join(lines[start:end])
            return hashlib.md5(node_content.encode()).hexdigest()
extract_python_elements(file_path)

Extract elements from Python file

Source code in toolboxv2/utils/extras/mkdocs.py
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
def extract_python_elements(self, file_path: Path) -> List[CodeElement]:
    """Extract elements from Python file"""
    elements = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        tree = ast.parse(content)

        for node in ast.walk(tree):
            if isinstance(node, ast.ClassDef):
                elements.append(CodeElement(
                    name=node.name,
                    element_type='class',
                    file_path=str(file_path),
                    line_start=node.lineno,
                    line_end=getattr(node, 'end_lineno', node.lineno),
                    signature=f"class {node.name}",
                    docstring=ast.get_docstring(node),
                    hash_signature=self._hash_node(node, content)
                ))

                # Extract methods
                for item in node.body:
                    if isinstance(item, ast.FunctionDef):
                        elements.append(CodeElement(
                            name=item.name,
                            element_type='method',
                            file_path=str(file_path),
                            line_start=item.lineno,
                            line_end=getattr(item, 'end_lineno', item.lineno),
                            signature=self._get_function_signature(item),
                            docstring=ast.get_docstring(item),
                            parent_class=node.name,
                            hash_signature=self._hash_node(item, content)
                        ))

            elif isinstance(node, ast.FunctionDef):
                elements.append(CodeElement(
                    name=node.name,
                    element_type='function',
                    file_path=str(file_path),
                    line_start=node.lineno,
                    line_end=getattr(node, 'end_lineno', node.lineno),
                    signature=self._get_function_signature(node),
                    docstring=ast.get_docstring(node),
                    hash_signature=self._hash_node(node, content)
                ))

    except Exception as e:
        logger.error(f"Error extracting Python elements from {file_path}: {e}")

    return elements
DocSection dataclass

Represents a documentation section with change tracking

Source code in toolboxv2/utils/extras/mkdocs.py
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
@dataclass
class DocSection:
    """Represents a documentation section with change tracking"""
    section_id: str
    file_path: str
    title: str
    content: str
    level: int
    line_start: int
    line_end: int
    source_refs: List[str] = field(default_factory=list)
    tags: List[str] = field(default_factory=list)
    hash_signature: str = ""
    content_hash: str = ""
    last_modified: datetime = field(default_factory=datetime.now)
    change_detected: bool = False
DocsAnalyzer

Analyzes documentation quality and completeness

Source code in toolboxv2/utils/extras/mkdocs.py
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
class DocsAnalyzer:
    """Analyzes documentation quality and completeness"""

    def __init__(self, index: DocsIndex, project_root: Path):
        self.index = index
        self.project_root = project_root

    def find_unclear_sections(self) -> List[str]:
        """Find sections with unclear or placeholder content"""
        unclear_indicators = [
            "todo", "fixme", "placeholder", "coming soon", "not implemented",
            "tbd", "under construction", "work in progress", "missing",
            "add content here", "fill this", "example here"
        ]

        unclear_sections = []

        for section_id, section in self.index.sections.items():
            content_lower = section.content.lower()

            # Check for unclear indicators
            if any(indicator in content_lower for indicator in unclear_indicators):
                unclear_sections.append(section_id)
                continue

            # Check for very short content (likely incomplete)
            if len(section.content.strip()) < 50:
                unclear_sections.append(section_id)
                continue

            # Check for sections with only code blocks (no explanation)
            code_blocks = re.findall(r'```[\s\S]*?```', section.content)
            text_without_code = re.sub(r'```[\s\S]*?```', '', section.content).strip()
            if code_blocks and len(text_without_code) < 20:
                unclear_sections.append(section_id)

        return unclear_sections

    def find_missing_implementations(self) -> List[Dict]:
        """Find TOC entries that don't have corresponding implementations"""
        missing = []

        for section_id, section in self.index.sections.items():
            # Look for function/class references that don't exist in code
            potential_refs = re.findall(r'`([A-Za-z_][A-Za-z0-9_.]*)`', section.content)

            for ref in potential_refs:
                if '.' in ref:  # Looks like a class.method reference
                    if not any(ref in element_id for element_id in self.index.code_elements.keys()):
                        missing.append({
                            "section_id": section_id,
                            "missing_ref": ref,
                            "type": "code_reference",
                            "content_context": section.content[:200] + "..."
                        })

        return missing
find_missing_implementations()

Find TOC entries that don't have corresponding implementations

Source code in toolboxv2/utils/extras/mkdocs.py
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
def find_missing_implementations(self) -> List[Dict]:
    """Find TOC entries that don't have corresponding implementations"""
    missing = []

    for section_id, section in self.index.sections.items():
        # Look for function/class references that don't exist in code
        potential_refs = re.findall(r'`([A-Za-z_][A-Za-z0-9_.]*)`', section.content)

        for ref in potential_refs:
            if '.' in ref:  # Looks like a class.method reference
                if not any(ref in element_id for element_id in self.index.code_elements.keys()):
                    missing.append({
                        "section_id": section_id,
                        "missing_ref": ref,
                        "type": "code_reference",
                        "content_context": section.content[:200] + "..."
                    })

    return missing
find_unclear_sections()

Find sections with unclear or placeholder content

Source code in toolboxv2/utils/extras/mkdocs.py
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
def find_unclear_sections(self) -> List[str]:
    """Find sections with unclear or placeholder content"""
    unclear_indicators = [
        "todo", "fixme", "placeholder", "coming soon", "not implemented",
        "tbd", "under construction", "work in progress", "missing",
        "add content here", "fill this", "example here"
    ]

    unclear_sections = []

    for section_id, section in self.index.sections.items():
        content_lower = section.content.lower()

        # Check for unclear indicators
        if any(indicator in content_lower for indicator in unclear_indicators):
            unclear_sections.append(section_id)
            continue

        # Check for very short content (likely incomplete)
        if len(section.content.strip()) < 50:
            unclear_sections.append(section_id)
            continue

        # Check for sections with only code blocks (no explanation)
        code_blocks = re.findall(r'```[\s\S]*?```', section.content)
        text_without_code = re.sub(r'```[\s\S]*?```', '', section.content).strip()
        if code_blocks and len(text_without_code) < 20:
            unclear_sections.append(section_id)

    return unclear_sections
DocsIndex dataclass

Complete documentation index with section-level tracking

Source code in toolboxv2/utils/extras/mkdocs.py
62
63
64
65
66
67
68
69
70
71
@dataclass
class DocsIndex:
    """Complete documentation index with section-level tracking"""
    sections: Dict[str, DocSection] = field(default_factory=dict)
    code_elements: Dict[str, CodeElement] = field(default_factory=dict)
    file_hashes: Dict[str, str] = field(default_factory=dict)
    section_hashes: Dict[str, str] = field(default_factory=dict)
    last_git_commit: Optional[str] = None
    last_indexed: datetime = field(default_factory=datetime.now)
    version: str = "1.1"
DocsIndexer

Main indexer that builds and maintains the complete documentation index

Source code in toolboxv2/utils/extras/mkdocs.py
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
class DocsIndexer:
    """Main indexer that builds and maintains the complete documentation index"""

    def __init__(self, project_root: Path, docs_root: Path,
                 include_dirs: List[str] = None, exclude_dirs: List[str] = None):
        self.project_root = project_root
        self.docs_root = docs_root
        self.git_detector = GitChangeDetector(project_root)
        self.import_analyzer = ImportAnalyzer(project_root)
        self.code_extractor = CodeElementExtractor()
        self.md_parser = MarkdownParser()
        self.index_file = docs_root / '.docs_index.json'

        # Directory filters
        self.include_dirs = include_dirs or ["toolboxv2", "src", "lib", "docs"]
        self.exclude_dirs = exclude_dirs or [
            "__pycache__", ".git", "node_modules", ".venv", "venv", "env",
            ".pytest_cache", ".mypy_cache", "dist", "build", ".tox",
            "coverage_html_report", ".coverage", ".next", ".nuxt",
            "target", "bin", "obj", ".gradle", ".idea", ".vscode"
        ]

    async def update_index_precise(self, current_index: DocsIndex,
                                   force_full_scan: bool = False,
                                   max_files_per_batch: int = 10) -> Tuple[DocsIndex, List[str], Dict[str, List[str]]]:
        """Update index with precise section-level tracking"""
        logger.info("Starting precise index update...")

        update_notes = []
        section_changes = {}  # file_path -> [changed_section_ids]

        if force_full_scan:
            return await self._full_scan_with_sections(current_index, max_files_per_batch)

        # Quick git change detection with timeout
        try:
            changes = await asyncio.wait_for(
                asyncio.get_event_loop().run_in_executor(
                    None, self.git_detector.get_changed_files, current_index.last_git_commit
                ),
                timeout=15.0
            )
        except asyncio.TimeoutError:
            logger.warning("Git detection timed out, using cached index")
            return current_index, ["Git timeout - using cached index"], {}

        # Process changes in batches
        changed_files = [c for c in changes if self._should_include_file(Path(c.file_path))]

        for i in range(0, len(changed_files), max_files_per_batch):
            batch = changed_files[i:i + max_files_per_batch]

            for change in batch:
                file_path = Path(change.file_path)

                if change.change_type == ChangeType.DELETED:
                    removed_sections = self._remove_file_from_index(current_index, str(file_path))
                    update_notes.append(f"Removed file: {file_path} ({len(removed_sections)} sections)")

                elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                    if not file_path.exists():
                        continue

                    # Quick hash check first
                    new_hash = self._get_file_hash(file_path)
                    old_hash = current_index.file_hashes.get(str(file_path))

                    if new_hash == old_hash:
                        continue  # No actual change

                    # Section-level update for markdown files
                    if file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                        changed_section_ids = await self._update_markdown_sections(
                            current_index, file_path
                        )
                        if changed_section_ids:
                            section_changes[str(file_path)] = changed_section_ids
                            update_notes.append(f"Updated {len(changed_section_ids)} sections in {file_path.name}")

                    # Code files - update entire file (faster for code)
                    elif file_path.suffix in ['.py', '.js', '.ts', '.jsx', '.tsx']:
                        self._update_file_in_index(current_index, file_path)
                        update_notes.append(f"Updated code file: {file_path}")

                    # Update file hash
                    current_index.file_hashes[str(file_path)] = new_hash

            # Yield control between batches
            await asyncio.sleep(0.01)

        # Update timestamps
        current_index.last_git_commit = self.git_detector.get_current_commit_hash()
        current_index.last_indexed = datetime.now()

        return current_index, update_notes, section_changes

    async def _update_markdown_sections(self, index: DocsIndex, file_path: Path) -> List[str]:
        """Update only changed sections in a markdown file"""
        try:
            # Get existing sections for this file
            existing_sections = {
                sid: section for sid, section in index.sections.items()
                if section.file_path == str(file_path)
            }

            # Parse with change detection
            new_sections, changed_section_ids = self.md_parser.parse_file_incremental(
                file_path, existing_sections
            )

            # Update only changed sections
            for section in new_sections:
                if section.change_detected or section.section_id not in index.sections:
                    index.sections[section.section_id] = section
                    # Update section hash tracking
                    index.section_hashes[section.section_id] = section.hash_signature

            # Remove sections that no longer exist
            current_section_ids = {s.section_id for s in new_sections}
            to_remove = [sid for sid in existing_sections.keys() if sid not in current_section_ids]

            for sid in to_remove:
                if sid in index.sections:
                    del index.sections[sid]
                if sid in index.section_hashes:
                    del index.section_hashes[sid]
                changed_section_ids.append(f"REMOVED:{sid}")

            return changed_section_ids

        except Exception as e:
            logger.error(f"Error updating markdown sections in {file_path}: {e}")
            return []

    def build_initial_index(self, file_extensions: List[str] = None) -> DocsIndex:
        """Build complete index for the first time with directory filtering"""
        logger.info("Building initial documentation index...")

        if file_extensions is None:
            file_extensions = ['.py', '.js', '.ts', '.jsx', '.tsx', '.md']

        index = DocsIndex()

        # Get current git commit
        index.last_git_commit = self.git_detector.get_current_commit_hash()

        # Get filtered file list
        target_files = self._get_filtered_files(file_extensions)

        logger.info(f"Processing {len(target_files)} files in {len(self.include_dirs)} directories")

        # Process each file type
        for file_path in target_files:
            logger.debug(f"Indexing {file_path}")
            try:
                if file_path.suffix == '.py':
                    self._index_python_file(file_path, index)
                elif file_path.suffix in ['.js', '.ts', '.jsx', '.tsx']:
                    self._index_js_file(file_path, index)
                elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                    self._index_md_file(file_path, index)

                # Store file hash for change detection
                index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

            except Exception as e:
                logger.error(f"Error indexing {file_path}: {e}")
                continue

        index.last_indexed = datetime.now()

        logger.info(f"Initial index built: {len(index.code_elements)} code elements, "
                    f"{len(index.sections)} doc sections, {len(index.import_refs)} import maps")

        return index

    def update_index(self, current_index: DocsIndex,
                     force_full_scan: bool = False) -> Tuple[DocsIndex, List[str]]:
        """Update index based on detected changes or force full scan"""
        logger.info("Updating documentation index...")

        update_notes = []

        if force_full_scan:
            logger.info("Performing force full scan...")
            # Get all current files
            current_files = set(self._get_filtered_files(['.py', '.js', '.ts', '.jsx', '.tsx', '.md']))

            # Check each file for changes
            for file_path in current_files:
                new_hash = self._get_file_hash(file_path)
                old_hash = current_index.file_hashes.get(str(file_path))

                if new_hash != old_hash:
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Updated (force scan): {file_path}")
                    current_index.file_hashes[str(file_path)] = new_hash

            # Remove files that no longer exist
            existing_files = {str(f) for f in current_files}
            to_remove = [f for f in current_index.file_hashes.keys() if f not in existing_files]

            for file_path in to_remove:
                self._remove_file_from_index(current_index, file_path)
                update_notes.append(f"Removed (no longer exists): {file_path}")

        else:
            # Git-based change detection
            changes = self.git_detector.get_changed_files(current_index.last_git_commit)

            for change in changes:
                file_path = Path(change.file_path)

                # Skip if not in our target directories
                if not self._should_include_file(file_path):
                    continue

                if change.change_type == ChangeType.DELETED:
                    self._remove_file_from_index(current_index, str(file_path))
                    update_notes.append(f"Removed (git): {file_path}")

                elif change.change_type == ChangeType.RENAMED:
                    if change.old_path:
                        self._remove_file_from_index(current_index, change.old_path)
                        update_notes.append(f"Removed (renamed from): {change.old_path}")

                    if file_path.exists():
                        self._update_file_in_index(current_index, file_path)
                        update_notes.append(f"Added (renamed to): {file_path}")
                        current_index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

                elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                    if not file_path.exists():
                        continue

                    # Verify actual change with hash comparison
                    new_hash = self._get_file_hash(file_path)
                    old_hash = current_index.file_hashes.get(str(file_path))

                    if new_hash != old_hash:
                        self._update_file_in_index(current_index, file_path)
                        update_notes.append(f"Updated (git {change.change_type.value}): {file_path}")
                        current_index.file_hashes[str(file_path)] = new_hash

        # Update git commit and timestamp
        current_index.last_git_commit = self.git_detector.get_current_commit_hash()
        current_index.last_indexed = datetime.now()

        if update_notes:
            logger.info(f"Index updated with {len(update_notes)} changes")
        else:
            logger.info("No changes detected in index update")

        return current_index, update_notes

    def _get_filtered_files(self, extensions: List[str]) -> List[Path]:
        """Get list of files matching extensions and directory filters"""
        files = []

        # If include_dirs is specified, only search in those directories
        search_dirs = []
        for include_dir in self.include_dirs:
            search_path = self.project_root / include_dir
            if search_path.exists() and search_path.is_dir():
                search_dirs.append(search_path)

        # If no include dirs exist, search entire project root
        if not search_dirs:
            search_dirs = [self.project_root]

        for search_dir in search_dirs:
            for ext in extensions:
                pattern = f"**/*{ext}"
                for file_path in search_dir.rglob(pattern):
                    if self._should_include_file(file_path):
                        files.append(file_path)

        return list(set(files))  # Remove duplicates

    def _should_include_file(self, file_path: Path) -> bool:
        """Check if file should be included based on directory filters"""
        file_str = str(file_path)

        # Check exclude patterns
        for exclude_dir in self.exclude_dirs:
            if exclude_dir in file_str:
                return False

        # If include_dirs specified, file must be in one of them
        if self.include_dirs:
            for include_dir in self.include_dirs:
                include_path = self.project_root / include_dir
                try:
                    if file_path.is_relative_to(include_path):
                        return True
                except ValueError:
                    continue
            return False

        return True

    def _update_file_in_index(self, index: DocsIndex, file_path: Path):
        """Update index for a specific file"""
        # Remove old entries first
        self._remove_file_from_index(index, str(file_path))

        # Add new entries based on file type
        try:
            if file_path.suffix == '.py':
                self._index_python_file(file_path, index)
            elif file_path.suffix in ['.js', '.ts', '.jsx', '.tsx']:
                self._index_js_file(file_path, index)
            elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                self._index_md_file(file_path, index)
        except Exception as e:
            logger.error(f"Error updating file {file_path}: {e}")

    def _index_python_file(self, file_path: Path, index: DocsIndex):
        """Index a Python file"""
        # Extract code elements
        elements = self.code_extractor.extract_python_elements(file_path)
        for element in elements:
            element_id = f"{element.file_path}:{element.name}"
            if element.parent_class:
                element_id = f"{element.file_path}:{element.parent_class}.{element.name}"
            index.code_elements[element_id] = element

        # Analyze imports
        imports = self.import_analyzer.analyze_python_imports(file_path)
        if imports:
            index.import_refs[str(file_path)] = imports

    def _index_js_file(self, file_path: Path, index: DocsIndex):
        """Index a JavaScript/TypeScript file"""
        imports = self.import_analyzer.analyze_js_imports(file_path)
        if imports:
            index.import_refs[str(file_path)] = imports

    def _index_md_file(self, file_path: Path, index: DocsIndex):
        """Index a markdown file"""
        sections = self.md_parser.parse_file(file_path)
        for section in sections:
            index.sections[section.section_id] = section

    def _get_file_hash(self, file_path: Path) -> str:
        """Get MD5 hash of file content"""
        try:
            with open(file_path, 'rb') as f:
                return hashlib.md5(f.read()).hexdigest()
        except Exception:
            return ""

    def _remove_file_from_index(self, index: DocsIndex, file_path: str):
        """Remove all references to a file from index"""
        # Remove code elements
        to_remove = [k for k, v in index.code_elements.items() if v.file_path == file_path]
        for key in to_remove:
            del index.code_elements[key]

        # Remove doc sections
        to_remove = [k for k, v in index.sections.items() if v.file_path == file_path]
        for key in to_remove:
            del index.sections[key]

        # Remove import refs
        if file_path in index.import_refs:
            del index.import_refs[file_path]

        # Remove file hash
        if file_path in index.file_hashes:
            del index.file_hashes[file_path]
build_initial_index(file_extensions=None)

Build complete index for the first time with directory filtering

Source code in toolboxv2/utils/extras/mkdocs.py
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
def build_initial_index(self, file_extensions: List[str] = None) -> DocsIndex:
    """Build complete index for the first time with directory filtering"""
    logger.info("Building initial documentation index...")

    if file_extensions is None:
        file_extensions = ['.py', '.js', '.ts', '.jsx', '.tsx', '.md']

    index = DocsIndex()

    # Get current git commit
    index.last_git_commit = self.git_detector.get_current_commit_hash()

    # Get filtered file list
    target_files = self._get_filtered_files(file_extensions)

    logger.info(f"Processing {len(target_files)} files in {len(self.include_dirs)} directories")

    # Process each file type
    for file_path in target_files:
        logger.debug(f"Indexing {file_path}")
        try:
            if file_path.suffix == '.py':
                self._index_python_file(file_path, index)
            elif file_path.suffix in ['.js', '.ts', '.jsx', '.tsx']:
                self._index_js_file(file_path, index)
            elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                self._index_md_file(file_path, index)

            # Store file hash for change detection
            index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

        except Exception as e:
            logger.error(f"Error indexing {file_path}: {e}")
            continue

    index.last_indexed = datetime.now()

    logger.info(f"Initial index built: {len(index.code_elements)} code elements, "
                f"{len(index.sections)} doc sections, {len(index.import_refs)} import maps")

    return index
update_index(current_index, force_full_scan=False)

Update index based on detected changes or force full scan

Source code in toolboxv2/utils/extras/mkdocs.py
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
def update_index(self, current_index: DocsIndex,
                 force_full_scan: bool = False) -> Tuple[DocsIndex, List[str]]:
    """Update index based on detected changes or force full scan"""
    logger.info("Updating documentation index...")

    update_notes = []

    if force_full_scan:
        logger.info("Performing force full scan...")
        # Get all current files
        current_files = set(self._get_filtered_files(['.py', '.js', '.ts', '.jsx', '.tsx', '.md']))

        # Check each file for changes
        for file_path in current_files:
            new_hash = self._get_file_hash(file_path)
            old_hash = current_index.file_hashes.get(str(file_path))

            if new_hash != old_hash:
                self._update_file_in_index(current_index, file_path)
                update_notes.append(f"Updated (force scan): {file_path}")
                current_index.file_hashes[str(file_path)] = new_hash

        # Remove files that no longer exist
        existing_files = {str(f) for f in current_files}
        to_remove = [f for f in current_index.file_hashes.keys() if f not in existing_files]

        for file_path in to_remove:
            self._remove_file_from_index(current_index, file_path)
            update_notes.append(f"Removed (no longer exists): {file_path}")

    else:
        # Git-based change detection
        changes = self.git_detector.get_changed_files(current_index.last_git_commit)

        for change in changes:
            file_path = Path(change.file_path)

            # Skip if not in our target directories
            if not self._should_include_file(file_path):
                continue

            if change.change_type == ChangeType.DELETED:
                self._remove_file_from_index(current_index, str(file_path))
                update_notes.append(f"Removed (git): {file_path}")

            elif change.change_type == ChangeType.RENAMED:
                if change.old_path:
                    self._remove_file_from_index(current_index, change.old_path)
                    update_notes.append(f"Removed (renamed from): {change.old_path}")

                if file_path.exists():
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Added (renamed to): {file_path}")
                    current_index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

            elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                if not file_path.exists():
                    continue

                # Verify actual change with hash comparison
                new_hash = self._get_file_hash(file_path)
                old_hash = current_index.file_hashes.get(str(file_path))

                if new_hash != old_hash:
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Updated (git {change.change_type.value}): {file_path}")
                    current_index.file_hashes[str(file_path)] = new_hash

    # Update git commit and timestamp
    current_index.last_git_commit = self.git_detector.get_current_commit_hash()
    current_index.last_indexed = datetime.now()

    if update_notes:
        logger.info(f"Index updated with {len(update_notes)} changes")
    else:
        logger.info("No changes detected in index update")

    return current_index, update_notes
update_index_precise(current_index, force_full_scan=False, max_files_per_batch=10) async

Update index with precise section-level tracking

Source code in toolboxv2/utils/extras/mkdocs.py
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
async def update_index_precise(self, current_index: DocsIndex,
                               force_full_scan: bool = False,
                               max_files_per_batch: int = 10) -> Tuple[DocsIndex, List[str], Dict[str, List[str]]]:
    """Update index with precise section-level tracking"""
    logger.info("Starting precise index update...")

    update_notes = []
    section_changes = {}  # file_path -> [changed_section_ids]

    if force_full_scan:
        return await self._full_scan_with_sections(current_index, max_files_per_batch)

    # Quick git change detection with timeout
    try:
        changes = await asyncio.wait_for(
            asyncio.get_event_loop().run_in_executor(
                None, self.git_detector.get_changed_files, current_index.last_git_commit
            ),
            timeout=15.0
        )
    except asyncio.TimeoutError:
        logger.warning("Git detection timed out, using cached index")
        return current_index, ["Git timeout - using cached index"], {}

    # Process changes in batches
    changed_files = [c for c in changes if self._should_include_file(Path(c.file_path))]

    for i in range(0, len(changed_files), max_files_per_batch):
        batch = changed_files[i:i + max_files_per_batch]

        for change in batch:
            file_path = Path(change.file_path)

            if change.change_type == ChangeType.DELETED:
                removed_sections = self._remove_file_from_index(current_index, str(file_path))
                update_notes.append(f"Removed file: {file_path} ({len(removed_sections)} sections)")

            elif change.change_type in [ChangeType.ADDED, ChangeType.MODIFIED]:
                if not file_path.exists():
                    continue

                # Quick hash check first
                new_hash = self._get_file_hash(file_path)
                old_hash = current_index.file_hashes.get(str(file_path))

                if new_hash == old_hash:
                    continue  # No actual change

                # Section-level update for markdown files
                if file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                    changed_section_ids = await self._update_markdown_sections(
                        current_index, file_path
                    )
                    if changed_section_ids:
                        section_changes[str(file_path)] = changed_section_ids
                        update_notes.append(f"Updated {len(changed_section_ids)} sections in {file_path.name}")

                # Code files - update entire file (faster for code)
                elif file_path.suffix in ['.py', '.js', '.ts', '.jsx', '.tsx']:
                    self._update_file_in_index(current_index, file_path)
                    update_notes.append(f"Updated code file: {file_path}")

                # Update file hash
                current_index.file_hashes[str(file_path)] = new_hash

        # Yield control between batches
        await asyncio.sleep(0.01)

    # Update timestamps
    current_index.last_git_commit = self.git_detector.get_current_commit_hash()
    current_index.last_indexed = datetime.now()

    return current_index, update_notes, section_changes
FileChange dataclass

Represents a file change detected by git

Source code in toolboxv2/utils/extras/mkdocs.py
75
76
77
78
79
80
81
82
@dataclass
class FileChange:
    """Represents a file change detected by git"""
    file_path: str
    change_type: ChangeType
    old_hash: Optional[str] = None
    new_hash: Optional[str] = None
    old_path: Optional[str] = None  # for renamed files
GitChangeDetector

Detects changes in the repository since last index update

Source code in toolboxv2/utils/extras/mkdocs.py
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
class GitChangeDetector:
    """Detects changes in the repository since last index update"""

    def __init__(self, repo_root: Path):
        self.repo_root = repo_root

    def get_current_commit_hash(self) -> Optional[str]:
        """Get current git commit hash"""
        try:
            result = subprocess.run(
                ["git", "rev-parse", "HEAD"],
                cwd=self.repo_root,
                capture_output=True,
                text=True
            )
            return result.stdout.strip() if result.returncode == 0 else None
        except Exception:
            return None

    def get_changed_files(self, since_commit: Optional[str] = None) -> List[FileChange]:
        """Get list of changed files since given commit with timeout"""
        changes = []

        try:
            if since_commit:
                cmd = ["git", "diff", "--name-status", f"{since_commit}..HEAD"]
            else:
                cmd = ["git", "ls-files"]

            # Add timeout to subprocess
            result = subprocess.run(
                cmd,
                cwd=self.repo_root,
                capture_output=True,
                text=True,
                timeout=20  # 20 second timeout for git operations
            )

            if result.returncode != 0:
                logger.warning(f"Git command failed with return code {result.returncode}")
                return changes

            # Process output with limits
            lines = result.stdout.strip().split('\n')[:1000]  # Limit to 1000 files

            for line in lines:
                if not line:
                    continue

                if since_commit:
                    parts = line.split('\t')
                    if len(parts) >= 2:
                        status = parts[0]
                        file_path = parts[1]

                        if status == 'A':
                            change_type = ChangeType.ADDED
                        elif status == 'M':
                            change_type = ChangeType.MODIFIED
                        elif status == 'D':
                            change_type = ChangeType.DELETED
                        elif status.startswith('R'):
                            change_type = ChangeType.RENAMED
                            old_path = parts[1] if len(parts) > 2 else None
                            file_path = parts[2] if len(parts) > 2 else parts[1]
                        else:
                            continue

                        changes.append(FileChange(
                            file_path=file_path,
                            change_type=change_type,
                            old_path=old_path if change_type == ChangeType.RENAMED else None
                        ))
                else:
                    changes.append(FileChange(
                        file_path=line.strip(),
                        change_type=ChangeType.ADDED
                    ))

        except subprocess.TimeoutExpired:
            logger.error("Git operation timed out after 20 seconds")
        except Exception as e:
            logger.error(f"Error detecting git changes: {e}")

        return changes
get_changed_files(since_commit=None)

Get list of changed files since given commit with timeout

Source code in toolboxv2/utils/extras/mkdocs.py
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def get_changed_files(self, since_commit: Optional[str] = None) -> List[FileChange]:
    """Get list of changed files since given commit with timeout"""
    changes = []

    try:
        if since_commit:
            cmd = ["git", "diff", "--name-status", f"{since_commit}..HEAD"]
        else:
            cmd = ["git", "ls-files"]

        # Add timeout to subprocess
        result = subprocess.run(
            cmd,
            cwd=self.repo_root,
            capture_output=True,
            text=True,
            timeout=20  # 20 second timeout for git operations
        )

        if result.returncode != 0:
            logger.warning(f"Git command failed with return code {result.returncode}")
            return changes

        # Process output with limits
        lines = result.stdout.strip().split('\n')[:1000]  # Limit to 1000 files

        for line in lines:
            if not line:
                continue

            if since_commit:
                parts = line.split('\t')
                if len(parts) >= 2:
                    status = parts[0]
                    file_path = parts[1]

                    if status == 'A':
                        change_type = ChangeType.ADDED
                    elif status == 'M':
                        change_type = ChangeType.MODIFIED
                    elif status == 'D':
                        change_type = ChangeType.DELETED
                    elif status.startswith('R'):
                        change_type = ChangeType.RENAMED
                        old_path = parts[1] if len(parts) > 2 else None
                        file_path = parts[2] if len(parts) > 2 else parts[1]
                    else:
                        continue

                    changes.append(FileChange(
                        file_path=file_path,
                        change_type=change_type,
                        old_path=old_path if change_type == ChangeType.RENAMED else None
                    ))
            else:
                changes.append(FileChange(
                    file_path=line.strip(),
                    change_type=ChangeType.ADDED
                ))

    except subprocess.TimeoutExpired:
        logger.error("Git operation timed out after 20 seconds")
    except Exception as e:
        logger.error(f"Error detecting git changes: {e}")

    return changes
get_current_commit_hash()

Get current git commit hash

Source code in toolboxv2/utils/extras/mkdocs.py
171
172
173
174
175
176
177
178
179
180
181
182
def get_current_commit_hash(self) -> Optional[str]:
    """Get current git commit hash"""
    try:
        result = subprocess.run(
            ["git", "rev-parse", "HEAD"],
            cwd=self.repo_root,
            capture_output=True,
            text=True
        )
        return result.stdout.strip() if result.returncode == 0 else None
    except Exception:
        return None
ImportAnalyzer

Analyzes imports and dependencies in Python and JS files

Source code in toolboxv2/utils/extras/mkdocs.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
class ImportAnalyzer:
    """Analyzes imports and dependencies in Python and JS files"""

    def __init__(self, project_root: Path):
        self.project_root = project_root

    def analyze_python_imports(self, file_path: Path) -> List[ImportReference]:
        """Analyze Python imports"""
        imports = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            tree = ast.parse(content)

            for node in ast.walk(tree):
                if isinstance(node, ast.Import):
                    for alias in node.names:
                        imports.append(ImportReference(
                            source_file=str(file_path),
                            target_file=self._resolve_python_import(alias.name),
                            import_name=alias.name,
                            line_number=node.lineno,
                            import_type='python'
                        ))

                elif isinstance(node, ast.ImportFrom):
                    module = node.module or ""
                    for alias in node.names:
                        target_file = self._resolve_python_import(f"{module}.{alias.name}")
                        imports.append(ImportReference(
                            source_file=str(file_path),
                            target_file=target_file,
                            import_name=f"{module}.{alias.name}",
                            line_number=node.lineno,
                            import_type='relative' if node.level > 0 else 'absolute'
                        ))

        except Exception as e:
            logger.error(f"Error analyzing Python imports in {file_path}: {e}")

        return imports

    def analyze_js_imports(self, file_path: Path) -> List[ImportReference]:
        """Analyze JavaScript/TypeScript imports"""
        imports = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            # Regex patterns for different import types
            patterns = [
                r'import\s+(?:{\s*([^}]+)\s*}|\*\s+as\s+(\w+)|(\w+))\s+from\s+["\']([^"\']+)["\']',
                r'const\s+(?:{\s*([^}]+)\s*}|(\w+))\s*=\s*require\(["\']([^"\']+)["\']\)',
                r'import\(["\']([^"\']+)["\']\)'
            ]

            for i, line in enumerate(content.split('\n'), 1):
                for pattern in patterns:
                    matches = re.finditer(pattern, line)
                    for match in matches:
                        groups = match.groups()
                        target = groups[-1]  # Last group is always the module path

                        imports.append(ImportReference(
                            source_file=str(file_path),
                            target_file=self._resolve_js_import(file_path.parent, target),
                            import_name=target,
                            line_number=i,
                            import_type='js'
                        ))

        except Exception as e:
            logger.error(f"Error analyzing JS imports in {file_path}: {e}")

        return imports

    def _resolve_python_import(self, import_name: str) -> str:
        """Resolve Python import to file path"""
        # Try to find the actual file
        parts = import_name.split('.')

        # Check in project root
        potential_paths = [
            self.project_root / '/'.join(parts) / '__init__.py',
            self.project_root / f"{'/'.join(parts)}.py",
        ]

        for path in potential_paths:
            if path.exists():
                return str(path)

        return import_name  # Return original if not found

    def _resolve_js_import(self, current_dir: Path, import_path: str) -> str:
        """Resolve JS import to file path"""
        if import_path.startswith('.'):
            # Relative import
            resolved = (current_dir / import_path).resolve()

            # Try different extensions
            for ext in ['.js', '.ts', '.jsx', '.tsx', '.json']:
                if resolved.with_suffix(ext).exists():
                    return str(resolved.with_suffix(ext))

            # Try index files
            for ext in ['.js', '.ts']:
                index_file = resolved / f"index{ext}"
                if index_file.exists():
                    return str(index_file)

        return import_path  # Return original if not found
analyze_js_imports(file_path)

Analyze JavaScript/TypeScript imports

Source code in toolboxv2/utils/extras/mkdocs.py
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
def analyze_js_imports(self, file_path: Path) -> List[ImportReference]:
    """Analyze JavaScript/TypeScript imports"""
    imports = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        # Regex patterns for different import types
        patterns = [
            r'import\s+(?:{\s*([^}]+)\s*}|\*\s+as\s+(\w+)|(\w+))\s+from\s+["\']([^"\']+)["\']',
            r'const\s+(?:{\s*([^}]+)\s*}|(\w+))\s*=\s*require\(["\']([^"\']+)["\']\)',
            r'import\(["\']([^"\']+)["\']\)'
        ]

        for i, line in enumerate(content.split('\n'), 1):
            for pattern in patterns:
                matches = re.finditer(pattern, line)
                for match in matches:
                    groups = match.groups()
                    target = groups[-1]  # Last group is always the module path

                    imports.append(ImportReference(
                        source_file=str(file_path),
                        target_file=self._resolve_js_import(file_path.parent, target),
                        import_name=target,
                        line_number=i,
                        import_type='js'
                    ))

    except Exception as e:
        logger.error(f"Error analyzing JS imports in {file_path}: {e}")

    return imports
analyze_python_imports(file_path)

Analyze Python imports

Source code in toolboxv2/utils/extras/mkdocs.py
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def analyze_python_imports(self, file_path: Path) -> List[ImportReference]:
    """Analyze Python imports"""
    imports = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        tree = ast.parse(content)

        for node in ast.walk(tree):
            if isinstance(node, ast.Import):
                for alias in node.names:
                    imports.append(ImportReference(
                        source_file=str(file_path),
                        target_file=self._resolve_python_import(alias.name),
                        import_name=alias.name,
                        line_number=node.lineno,
                        import_type='python'
                    ))

            elif isinstance(node, ast.ImportFrom):
                module = node.module or ""
                for alias in node.names:
                    target_file = self._resolve_python_import(f"{module}.{alias.name}")
                    imports.append(ImportReference(
                        source_file=str(file_path),
                        target_file=target_file,
                        import_name=f"{module}.{alias.name}",
                        line_number=node.lineno,
                        import_type='relative' if node.level > 0 else 'absolute'
                    ))

    except Exception as e:
        logger.error(f"Error analyzing Python imports in {file_path}: {e}")

    return imports
ImportReference dataclass

Represents an import/dependency reference

Source code in toolboxv2/utils/extras/mkdocs.py
85
86
87
88
89
90
91
92
@dataclass
class ImportReference:
    """Represents an import/dependency reference"""
    source_file: str
    target_file: str
    import_name: str
    line_number: int
    import_type: str  # 'python', 'js', 'relative', 'absolute'
MarkdownDocsSystem

Production-ready unified markdown documentation system

Source code in toolboxv2/utils/extras/mkdocs.py
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
class MarkdownDocsSystem:
    """Production-ready unified markdown documentation system"""

    def __init__(self, app: AppType, docs_root: str = "../docs", source_root: str = ".",
                 include_dirs: List[str] = None, exclude_dirs: List[str] = None):
        self.app = app
        self.docs_root = Path(docs_root)
        self.source_root = Path(source_root)
        self.project_root = Path.cwd()

        # Directory filters
        self.include_dirs = include_dirs or ["toolboxv2", "flows", "mods", "utils", "tbjs", "tests", "tcm", "docs"]
        self.exclude_dirs = exclude_dirs or [
            "__pycache__", ".git", "node_modules", ".venv", "venv", "env", "python_env",
            ".pytest_cache", ".mypy_cache", "dist", "build", ".tox", "coverage_html_report",
            ".coverage", ".next", ".nuxt", "target", "bin", "obj", ".gradle", ".idea",
            ".vscode", "temp", "tmp", "logs", ".cache", "coverage", ".data", ".config",
            ".info", "web", "simple-core", "src-core"
        ]

        # Index management
        self.index_file = self.docs_root / '.docs_index.json'
        self.current_index: Optional[DocsIndex] = None

        # Ensure directories exist
        self.docs_root.mkdir(exist_ok=True)

        # Internal cache for performance
        self._search_cache = {}
        self._cache_timeout = 300  # 5 minutes

    # ==================== CORE INDEX MANAGEMENT ====================


    def _load_index(self, minimal: bool = False, force_reload: bool = False) -> DocsIndex:
        """Unified index loading with proper incremental loading"""
        # Return cached index if available and not forcing reload
        if not force_reload and self.current_index and not minimal:
            return self.current_index

        if not self.index_file.exists():
            logger.info("No index file found, will build new index")
            return DocsIndex()

        try:
            with open(self.index_file, 'r', encoding='utf-8') as f:
                data = json.load(f)

            index = DocsIndex()
            index.last_indexed = datetime.fromisoformat(
                data.get('last_indexed', datetime.now().isoformat())
            )
            index.last_git_commit = data.get('last_git_commit')
            index.version = data.get('version', '1.1')
            index.file_hashes = data.get('file_hashes', {})
            index.section_hashes = data.get('section_hashes', {})

            # Load sections with optional truncation for performance
            sections_data = data.get('sections', {})
            section_limit = 200 if minimal else None
            section_count = 0

            for section_id, section_data in sections_data.items():
                if minimal and section_count >= section_limit:
                    break

                content = section_data['content']
                if minimal:
                    content = content[:800]  # Truncate for speed

                index.sections[section_id] = DocSection(
                    section_id=section_data['section_id'],
                    file_path=section_data['file_path'],
                    title=section_data['title'],
                    content=content,
                    level=section_data['level'],
                    line_start=section_data['line_start'],
                    line_end=section_data['line_end'],
                    source_refs=section_data.get('source_refs', [])[:5 if minimal else None],
                    tags=section_data.get('tags', [])[:3 if minimal else None],
                    hash_signature=section_data.get('hash_signature', ''),
                    content_hash=section_data.get('content_hash', ''),
                    last_modified=datetime.fromisoformat(
                        section_data.get('last_modified', datetime.now().isoformat()))
                )
                section_count += 1

            # Load code elements only if not minimal
            if not minimal:
                for element_id, element_data in data.get('code_elements', {}).items():
                    index.code_elements[element_id] = CodeElement(
                        name=element_data['name'],
                        element_type=element_data['element_type'],
                        file_path=element_data['file_path'],
                        line_start=element_data['line_start'],
                        line_end=element_data['line_end'],
                        signature=element_data['signature'],
                        docstring=element_data.get('docstring'),
                        hash_signature=element_data.get('hash_signature', ''),
                        parent_class=element_data.get('parent_class')
                    )

            # Cache full index for future use
            if not minimal:
                self.current_index = index

            logger.info(
                f"Loaded {'minimal' if minimal else 'full'} index: {len(index.sections)} sections, {len(index.code_elements)} elements")
            return index

        except Exception as e:
            logger.error(f"Error loading index: {e}")
            return DocsIndex()

    def _save_index(self, index: DocsIndex):
        """Optimized index saving"""
        try:
            data = {
                'version': index.version,
                'last_git_commit': index.last_git_commit,
                'last_indexed': index.last_indexed.isoformat(),
                'file_hashes': index.file_hashes,
                'section_hashes': index.section_hashes,
                'sections': {},
                'code_elements': {}
            }

            # Convert sections efficiently
            for section_id, section in index.sections.items():
                section_dict = asdict(section)
                section_dict['last_modified'] = section.last_modified.isoformat()
                data['sections'][section_id] = section_dict

            # Convert code elements efficiently
            for element_id, element in index.code_elements.items():
                data['code_elements'][element_id] = asdict(element)

            # Write with minimal formatting for speed
            with open(self.index_file, 'w', encoding='utf-8') as f:
                json.dump(data, f, separators=(',', ':'), ensure_ascii=False)

        except Exception as e:
            logger.error(f"Error saving index: {e}")

    async def _save_index_async(self, index: DocsIndex):
        """Async wrapper for index saving"""
        try:
            await asyncio.get_event_loop().run_in_executor(None, self._save_index, index)
        except Exception as e:
            logger.error(f"Async index save failed: {e}")

    # ==================== CORE FUNCTIONALITY ====================

    async def docs_reader(self,
                          query: Optional[str] = None,
                          section_id: Optional[str] = None,
                          file_path: Optional[str] = None,
                          tags: Optional[List[str]] = None,
                          include_source_refs: bool = True,
                          format_type: str = "structured",
                          max_results: int = 25) -> Result:
        """Ultra-fast unified docs reader with proper index loading"""
        try:
            start_time = time.time()

            # Load index if needed - try cached first, then saved, then build
            if not self.current_index:
                # First try to load from saved file
                self.current_index = self._load_index(minimal=False)

                # If no sections found, suggest running initial_docs_parse
                if not self.current_index.sections:
                    if self.index_file.exists():
                        logger.warning("Index file exists but contains no documentation sections")
                        # Try to find any markdown files
                        md_files = list(self.docs_root.rglob('*.md'))
                        if md_files:
                            logger.info(f"Found {len(md_files)} markdown files, re-parsing...")
                            # Re-parse markdown files
                            for md_file in md_files[:10]:  # Limit for quick check
                                sections = self._parse_markdown_file(md_file)
                                for section in sections:
                                    self.current_index.sections[section.section_id] = section

                            if self.current_index.sections:
                                logger.info(f"Re-parsed {len(self.current_index.sections)} sections")
                                await self._save_index_async(self.current_index)

                    if not self.current_index.sections:
                        return Result.default_user_error(
                            "No documentation sections found. Run initial_docs_parse() to build the documentation index, "
                            "or create some .md files in the docs directory."
                        )

            # Check cache for repeated queries
            cache_key = f"{query}:{section_id}:{file_path}:{tags}:{format_type}:{max_results}"
            if cache_key in self._search_cache:
                cache_entry = self._search_cache[cache_key]
                if time.time() - cache_entry['timestamp'] < self._cache_timeout:
                    return Result.ok(cache_entry['result'])

            # Fast path for specific section
            if section_id:
                if section_id in self.current_index.sections:
                    section = self.current_index.sections[section_id]
                    result = self._format_single_section(section, include_source_refs, format_type)
                    return result
                else:
                    return Result.default_user_error(f"Section not found: {section_id}")

            # Search and format results
            matching_sections = await self._search_sections(query, file_path, tags, max_results, start_time)
            result_data = self._format_sections(matching_sections, include_source_refs, format_type)

            # Cache the result
            self._search_cache[cache_key] = {
                'result': result_data,
                'timestamp': time.time()
            }

            return Result.ok(result_data)

        except Exception as e:
            logger.error(f"Error in docs_reader: {e}")
            return Result.default_user_error(f"Error reading documentation: {e}")

    async def docs_writer(self,
                          action: str,
                          file_path: Optional[str] = None,
                          section_title: Optional[str] = None,
                          content: Optional[str] = None,
                          source_file: Optional[str] = None,
                          auto_generate: bool = False,
                          position: Optional[str] = None,
                          level: int = 2) -> Result:
        """Unified optimized docs writer"""
        try:
            if not self.current_index:
                self.current_index = self._load_index(minimal=True)

            result = {"action": action, "timestamp": datetime.now().isoformat()}

            if action == "update_section":
                return await self._update_section(file_path, section_title, content,
                                                  source_file, auto_generate)
            elif action == "add_section":
                return await self._add_section(file_path, section_title, content,
                                               source_file, auto_generate, position, level)
            elif action == "create_file":
                return await self._create_file(file_path, content, source_file, auto_generate)
            elif action == "generate_from_code":
                return await self._generate_from_code(source_file, file_path, auto_generate)
            else:
                return Result.default_user_error(f"Unknown action: {action}")

        except Exception as e:
            logger.error(f"Error in docs_writer: {e}")
            return Result.default_user_error(f"Error writing documentation: {e}")

    # ==================== ADVANCED OPERATIONS ====================

    async def get_update_suggestions(self, force_scan: bool = False,
                                     max_suggestions: int = 50) -> Result:
        """Get prioritized documentation update suggestions"""
        try:
            if not self.current_index:
                self.current_index = self._load_index()

            # Quick scan for changes if needed
            if force_scan:
                await self._quick_update_index()

            suggestions = []

            # Find undocumented code elements
            undocumented = self._find_undocumented_elements()
            for element in undocumented[:max_suggestions // 2]:
                priority = self._assess_priority(element)
                suggestions.append({
                    "type": "missing_documentation",
                    "element_name": element.name,
                    "element_type": element.element_type,
                    "file_path": element.file_path,
                    "priority": priority,
                    "action": "generate_from_code" if element.element_type == "class" else "add_section"
                })

            # Find unclear sections
            unclear = self._find_unclear_sections()
            for section_id in unclear[:max_suggestions // 2]:
                section = self.current_index.sections[section_id]
                suggestions.append({
                    "type": "unclear_documentation",
                    "section_id": section_id,
                    "title": section.title,
                    "file_path": section.file_path,
                    "priority": "medium",
                    "action": "update_section"
                })

            # Sort by priority
            priority_order = {"high": 0, "medium": 1, "low": 2}
            suggestions.sort(key=lambda x: priority_order.get(x["priority"], 3))

            return Result.ok({
                "suggestions": suggestions[:max_suggestions],
                "total_found": len(suggestions),
                "undocumented_elements": len(undocumented),
                "unclear_sections": len(unclear)
            })

        except Exception as e:
            logger.error(f"Error getting update suggestions: {e}")
            return Result.default_user_error(f"Error analyzing updates: {e}")

    async def auto_update_docs(self, dry_run: bool = False, max_updates: int = 5,
                               timeout: int = 30) -> Result:
        """Automatically update documentation with timeout protection"""
        try:
            # Get suggestions
            suggestions_result = await asyncio.wait_for(
                self.get_update_suggestions(max_suggestions=max_updates * 2),
                timeout=10.0
            )

            if suggestions_result.is_error():
                return suggestions_result

            suggestions = suggestions_result.get("suggestions")[:max_updates]

            if dry_run:
                return Result.ok({
                    "dry_run": True,
                    "would_update": len(suggestions),
                    "suggestions": [s["type"] + ": " + s.get("element_name", s.get("title", ""))
                                    for s in suggestions]
                })

            results = []
            start_time = time.time()

            for suggestion in suggestions:
                if time.time() - start_time > timeout:
                    break

                try:
                    if suggestion["action"] == "generate_from_code":
                        result = await asyncio.wait_for(
                            self.docs_writer(
                                action="generate_from_code",
                                source_file=suggestion["file_path"],
                                auto_generate=True
                            ), timeout=10.0
                        )
                    elif suggestion["action"] == "add_section":
                        result = await asyncio.wait_for(
                            self.docs_writer(
                                action="add_section",
                                file_path=f"{Path(suggestion['file_path']).stem}.md",
                                section_title=suggestion["element_name"],
                                source_file=suggestion["file_path"],
                                auto_generate=True
                            ), timeout=8.0
                        )
                    else:
                        continue

                    results.append({
                        "suggestion": suggestion["type"],
                        "status": "success" if result.is_ok() else "error",
                        "details": str(result.error) if result.is_error() else "completed"
                    })

                except asyncio.TimeoutError:
                    results.append({
                        "suggestion": suggestion["type"],
                        "status": "timeout",
                        "details": "Operation timed out"
                    })

                await asyncio.sleep(0.1)  # Brief pause between operations

            return Result.ok({
                "processed": len(results),
                "successful": len([r for r in results if r["status"] == "success"]),
                "results": results,
                "execution_time": f"{time.time() - start_time:.2f}s"
            })

        except Exception as e:
            logger.error(f"Error in auto_update_docs: {e}")
            return Result.default_user_error(f"Auto-update failed: {e}")

    async def initial_docs_parse(self, update_index: bool = True, force_rebuild: bool = False) -> Result:
        """Parse existing documentation and build initial index with proper saving"""
        try:
            logger.info("Starting initial documentation parse...")

            # Check if we can use existing index
            if not force_rebuild and self.index_file.exists() and not update_index:
                self.current_index = self._load_index(minimal=False)
                if self.current_index.sections or self.current_index.code_elements:
                    logger.info("Using existing index")
                    return Result.ok({
                        "total_sections": len(self.current_index.sections),
                        "total_code_elements": len(self.current_index.code_elements),
                        "linked_sections": len([s for s in self.current_index.sections.values() if s.source_refs]),
                        "completion_rate": f"{(len([s for s in self.current_index.sections.values() if s.source_refs]) / max(len(self.current_index.sections), 1) * 100):.1f}%",
                        "used_cached": True
                    })

            if update_index or force_rebuild:
                # Build comprehensive index
                logger.info("Building comprehensive index from source files...")
                self.current_index = await asyncio.get_event_loop().run_in_executor(
                    None, self._build_full_index
                )
            else:
                self.current_index = self._load_index(minimal=False)

            # Ensure we have some documentation sections
            if not self.current_index.sections:
                logger.info("No documentation sections found, scanning for markdown files...")
                # Look for markdown files in docs and project root
                search_paths = [self.docs_root]
                if self.docs_root != self.project_root:
                    search_paths.append(self.project_root)

                for search_path in search_paths:
                    for md_file in search_path.rglob('*.md'):
                        if self._should_include_file(md_file):
                            sections = self._parse_markdown_file(md_file)
                            for section in sections:
                                self.current_index.sections[section.section_id] = section

                    # Don't search too deep
                    if len(self.current_index.sections) > 0:
                        break

                logger.info(f"Found {len(self.current_index.sections)} documentation sections")

            # Link documentation sections to code elements
            linked_count = await self._link_docs_to_code()

            # Save updated index with proper error handling
            try:
                await self._save_index_async(self.current_index)
                logger.info("Index saved successfully")
            except Exception as e:
                logger.error(f"Failed to save index: {e}")
                # Continue anyway, we have the index in memory

            completion_rate = (
                    linked_count / max(len(self.current_index.sections), 1) * 100) if self.current_index.sections else 0

            return Result.ok({
                "total_sections": len(self.current_index.sections),
                "total_code_elements": len(self.current_index.code_elements),
                "linked_sections": linked_count,
                "completion_rate": f"{completion_rate:.1f}%",
                "index_file": str(self.index_file),
                "docs_root": str(self.docs_root)
            })

        except Exception as e:
            logger.error(f"Error in initial_docs_parse: {e}")
            return Result.default_user_error(f"Failed to parse docs: {e}")

    # ==================== INTERNAL HELPER METHODS ====================

    async def _search_sections(self, query: Optional[str], file_path: Optional[str],
                               tags: Optional[List[str]], max_results: int, start_time: float) -> List[DocSection]:
        """Optimized section search with early termination"""
        matching_sections = []
        search_terms = set(query.lower().split()) if query else set()

        for section in self.current_index.sections.values():
            if len(matching_sections) >= max_results:
                break

            # Timeout protection
            if time.time() - start_time > 3.0:
                break

            # Quick filters
            if file_path and file_path not in section.file_path:
                continue

            if tags and not any(tag in section.tags for tag in tags):
                continue

            if search_terms:
                search_content = f"{section.title} {section.content[:200]}".lower()
                if not any(term in search_content for term in search_terms):
                    continue

            matching_sections.append(section)

        return matching_sections

    def _format_sections(self, sections: List[DocSection], include_source_refs: bool,
                         format_type: str) -> dict:
        """Unified section formatting"""
        if format_type == "markdown":
            output = []
            for section in sections[:20]:
                output.append(f"{'#' * section.level} {section.title}")
                output.append("")
                content = section.content[:500] + ("..." if len(section.content) > 500 else "")
                output.append(content)
                if include_source_refs and section.source_refs:
                    output.append(f"\n**References:** {', '.join(section.source_refs[:3])}")
                output.append("")
            return "\n".join(output)

        elif format_type == "json":
            return [self._section_to_dict(s, include_source_refs) for s in sections[:20]]

        else:  # structured
            return {
                "sections": [self._section_to_dict(s, include_source_refs) for s in sections[:20]],
                "metadata": {
                    "total_available": len(sections),
                    "returned_sections": min(len(sections), 20),
                    "truncated": len(sections) > 20
                }
            }

    def _format_single_section(self, section: DocSection, include_source_refs: bool,
                               format_type: str) -> Result:
        """Format a single section efficiently"""
        if format_type == "markdown":
            content = f"{'#' * section.level} {section.title}\n\n{section.content}"
            if include_source_refs and section.source_refs:
                content += f"\n\n**References:** {', '.join(section.source_refs[:5])}"
            return Result.ok(content)

        elif format_type == "json":
            return Result.ok([self._section_to_dict(section, include_source_refs)])

        else:  # structured
            return Result.ok({
                "sections": [self._section_to_dict(section, include_source_refs)],
                "metadata": {"total_sections": 1, "query_type": "specific_section"}
            })

    def _section_to_dict(self, section: DocSection, include_source_refs: bool) -> dict:
        """Convert section to dictionary efficiently"""
        data = {
            "id": section.section_id,
            "title": section.title,
            "content": section.content[:500],  # Limit for performance
            "file_path": section.file_path,
            "level": section.level,
            "tags": section.tags[:3]
        }
        if include_source_refs:
            data["source_refs"] = section.source_refs[:3]
        return data

    async def _update_section(self, file_path: str, section_title: str, content: str,
                              source_file: str, auto_generate: bool) -> Result:
        """Update existing section"""
        if not file_path or not section_title:
            return Result.default_user_error("file_path and section_title required")

        section_id = f"{Path(file_path).name}#{section_title}"
        if section_id not in self.current_index.sections:
            return Result.default_user_error(f"Section not found: {section_id}")

        section = self.current_index.sections[section_id]

        if auto_generate and source_file:
            content = await self._generate_content(section_title, source_file)

        if not content:
            return Result.default_user_error("content required")

        # Update file
        success = self._update_section_in_file(section, content)
        if not success:
            return Result.default_user_error("Failed to update file")

        # Update index
        section.content = content
        section.content_hash = hashlib.md5(content.encode()).hexdigest()
        section.last_modified = datetime.now()

        asyncio.create_task(self._save_index_async(self.current_index))

        return Result.ok({"action": "section_updated", "section_id": section_id})

    async def _add_section(self, file_path: str, section_title: str, content: str,
                           source_file: str, auto_generate: bool, position: str, level: int) -> Result:
        """Add new section to file"""
        if not file_path or not section_title:
            return Result.default_user_error("file_path and section_title required")

        if auto_generate and source_file:
            content = await self._generate_content(section_title, source_file)

        if not content:
            content = f"Content for {section_title}\n\nTODO: Add documentation here."

        # Create section
        section = DocSection(
            section_id=f"{Path(file_path).name}#{section_title}",
            file_path=str(self.docs_root / file_path),
            title=section_title,
            content=content,
            level=level,
            line_start=0,
            line_end=0,
            hash_signature=hashlib.md5(content.encode()).hexdigest(),
            content_hash=hashlib.md5(content.encode()).hexdigest()
        )

        # Add to file
        success = self._add_section_to_file(file_path, section, position)
        if not success:
            return Result.default_user_error("Failed to add section to file")

        # Update index
        self.current_index.sections[section.section_id] = section
        asyncio.create_task(self._save_index_async(self.current_index))

        return Result.ok({"action": "section_added", "section_id": section.section_id})

    async def _create_file(self, file_path: str, content: str, source_file: str,
                           auto_generate: bool) -> Result:
        """Create new documentation file"""
        if not file_path:
            return Result.default_user_error("file_path required")

        full_path = self.docs_root / file_path

        if full_path.exists():
            return Result.default_user_error(f"File already exists: {file_path}")

        if auto_generate and source_file:
            content = await self._generate_file_content(source_file)

        if not content:
            title = Path(file_path).stem.replace('_', ' ').title()
            content = f"# {title}\n\nDocumentation for {title}.\n\n"

        full_path.parent.mkdir(parents=True, exist_ok=True)
        full_path.write_text(content, encoding='utf-8')

        return Result.ok({"action": "file_created", "file_path": str(full_path)})

    async def _generate_from_code(self, source_file: str, file_path: str,
                                  auto_generate: bool) -> Result:
        """Generate documentation from source code"""
        if not source_file:
            return Result.default_user_error("source_file required")

        if not Path(source_file).exists():
            return Result.default_user_error(f"Source file not found: {source_file}")

        if not file_path:
            file_path = f"{Path(source_file).stem}.md"

        content = await self._generate_file_content(source_file) if auto_generate else ""

        return await self._create_file(file_path, content, source_file, False)

    async def _generate_content(self, title: str, source_file: str) -> str:
        """Generate content using AI with timeout protection"""
        try:
            isaa = self.app.get_mod("isaa")
            agent = await isaa.get_agent("docwriter")

            if not agent:
                return f"# {title}\n\nAI agent not available. Please add content manually."

            with open(source_file, 'r', encoding='utf-8') as f:
                source_content = f.read()[:2000]  # Limit source size

            prompt = f"""Generate concise documentation for "{title}" from this code:

```python
{source_content}
```

Requirements: Clear, concise, markdown format, no section header."""

            result = await asyncio.wait_for(
                agent.a_run_llm_completion(
                    messages=[{"role": "user", "content": prompt}],
                    model_preference="fast"
                ), timeout=8.0
            )
            return result

        except Exception as e:
            logger.error(f"Error generating content: {e}")
            return f"# {title}\n\nError generating content: {e}"

    async def _generate_file_content(self, source_file: str) -> str:
        """Generate complete file documentation"""
        try:
            isaa = self.app.get_mod("isaa")
            agent = await isaa.get_agent("docwriter")

            if not agent:
                return f"# {Path(source_file).stem}\n\nAI agent not available."

            # Extract code elements
            elements = self._extract_code_elements(Path(source_file))

            with open(source_file, 'r', encoding='utf-8') as f:
                source_content = f.read()[:3000]  # Limit source size

            prompt = f"""Generate comprehensive documentation for: {source_file}

Code:
```python
{source_content}
```

Elements: {len(elements)} classes/functions found

Create complete markdown with headers, overview, and examples."""

            result = await asyncio.wait_for(
                agent.a_run_llm_completion(
                    messages=[{"role": "user", "content": prompt}],
                    model_preference="fast"
                ), timeout=12.0
            )
            return result

        except Exception as e:
            logger.error(f"Error generating file content: {e}")
            return f"# {Path(source_file).stem}\n\nError generating documentation: {e}"

    def _build_full_index(self) -> DocsIndex:
        """Build comprehensive index from scratch with better progress tracking"""
        logger.info("Building full documentation index...")

        index = DocsIndex()
        index.last_git_commit = self._get_git_commit()

        # Get target files
        target_files = self._get_target_files()
        logger.info(f"Processing {len(target_files)} files")

        code_files_processed = 0
        md_files_processed = 0

        for i, file_path in enumerate(target_files):
            try:
                if file_path.suffix == '.py':
                    elements = self._extract_code_elements(file_path)
                    for element in elements:
                        element_id = f"{element.file_path}:{element.name}"
                        if element.parent_class:
                            element_id = f"{element.file_path}:{element.parent_class}.{element.name}"
                        index.code_elements[element_id] = element
                    code_files_processed += 1

                elif file_path.suffix.lower() == '.md':
                    sections = self._parse_markdown_file(file_path)
                    for section in sections:
                        index.sections[section.section_id] = section
                    if sections:
                        md_files_processed += 1

                # Store file hash
                index.file_hashes[str(file_path)] = self._get_file_hash(file_path)

                # Progress logging
                if i % 50 == 0 and i > 0:
                    logger.info(f"Progress: {i}/{len(target_files)} files processed")

            except Exception as e:
                logger.error(f"Error processing {file_path}: {e}")

        index.last_indexed = datetime.now()
        logger.info(f"Index built: {len(index.code_elements)} elements from {code_files_processed} Python files, "
                    f"{len(index.sections)} sections from {md_files_processed} markdown files")

        return index

    async def _quick_update_index(self):
        """Quick index update for changed files only"""
        try:
            changes = await asyncio.get_event_loop().run_in_executor(
                None, self._get_git_changes
            )

            for change in changes[:20]:  # Limit changes processed
                file_path = Path(change)
                if not self._should_include_file(file_path):
                    continue

                if not file_path.exists():
                    self._remove_file_from_index(file_path)
                    continue

                new_hash = self._get_file_hash(file_path)
                old_hash = self.current_index.file_hashes.get(str(file_path))

                if new_hash != old_hash:
                    self._update_file_in_index(file_path)
                    self.current_index.file_hashes[str(file_path)] = new_hash

            self.current_index.last_indexed = datetime.now()

        except Exception as e:
            logger.error(f"Error in quick index update: {e}")

    async def _link_docs_to_code(self) -> int:
        """Link documentation sections to code elements"""
        linked_count = 0

        for section_id, section in self.current_index.sections.items():
            matches = self._find_code_matches(section)
            if matches:
                section.source_refs.extend(matches)
                section.source_refs = list(set(section.source_refs))  # Remove duplicates
                linked_count += 1

        return linked_count

    def _find_code_matches(self, section: DocSection) -> List[str]:
        """Find code elements that match a documentation section"""
        matches = []
        title_words = set(re.findall(r'\b[A-Za-z_][A-Za-z0-9_]*\b', section.title.lower()))
        content_words = set(re.findall(r'\b[A-Za-z_][A-Za-z0-9_]*\b', section.content.lower()))

        for element_id, element in self.current_index.code_elements.items():
            score = 0

            # Direct matches
            if element.name.lower() in title_words:
                score += 10
            if element.name.lower() in content_words:
                score += 5

            # File correlation
            if Path(element.file_path).stem.lower() in Path(section.file_path).stem.lower():
                score += 3

            if score >= 5:
                matches.append(element_id)

        return matches

    def _find_undocumented_elements(self) -> List[CodeElement]:
        """Find code elements without documentation"""
        undocumented = []

        for element_id, element in self.current_index.code_elements.items():
            has_docs = any(
                element_id in section.source_refs or
                element.name in section.content or
                element.name in section.title
                for section in self.current_index.sections.values()
            )

            if not has_docs:
                undocumented.append(element)

        return undocumented

    def _find_unclear_sections(self) -> List[str]:
        """Find sections with unclear content"""
        unclear = []
        unclear_indicators = ['todo', 'fixme', 'placeholder', 'coming soon', 'tbd']

        for section_id, section in self.current_index.sections.items():
            content_lower = section.content.lower()

            if (any(indicator in content_lower for indicator in unclear_indicators) or
                len(section.content.strip()) < 50):
                unclear.append(section_id)

        return unclear

    def _assess_priority(self, element: CodeElement) -> str:
        """Assess documentation priority for code element"""
        score = 0

        # Type weights
        type_scores = {'class': 5, 'function': 3, 'method': 2}
        score += type_scores.get(element.element_type, 1)

        # Complexity
        if element.signature and element.signature.count(',') > 2:
            score += 2

        # Public vs private
        if not element.name.startswith('_'):
            score += 2

        # No docstring
        if not element.docstring:
            score += 2

        return "high" if score >= 8 else "medium" if score >= 5 else "low"

    # ==================== FILE OPERATIONS ====================

    def _update_section_in_file(self, section: DocSection, new_content: str) -> bool:
        """Update section content in file"""
        try:
            file_path = Path(section.file_path)
            with open(file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Replace section content
            new_section = f"{'#' * section.level} {section.title}\n\n{new_content}\n\n"
            new_lines = new_section.split('\n')

            # Update the lines
            lines[section.line_start:section.line_end + 1] = [line + '\n' for line in new_lines]

            with open(file_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return True
        except Exception as e:
            logger.error(f"Error updating section in file: {e}")
            return False

    def _add_section_to_file(self, file_path: str, section: DocSection, position: str) -> bool:
        """Add section to file"""
        try:
            full_path = self.docs_root / file_path

            if not full_path.exists():
                # Create new file
                content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
                full_path.write_text(content, encoding='utf-8')
                return True

            # Add to existing file
            with open(full_path, 'r', encoding='utf-8') as f:
                content = f.read()

            new_section = f"\n{'#' * section.level} {section.title}\n\n{section.content}\n\n"

            if position == "top":
                content = new_section + content
            else:
                content = content + new_section

            full_path.write_text(content, encoding='utf-8')
            return True

        except Exception as e:
            logger.error(f"Error adding section to file: {e}")
            return False

    # ==================== UTILITY METHODS ====================

    def _get_target_files(self) -> List[Path]:
        """Get filtered list of target files"""
        files = []
        extensions = ['.py', '.md']

        search_dirs = [self.project_root / d for d in self.include_dirs if (self.project_root / d).exists()]
        if not search_dirs:
            search_dirs = [self.project_root]

        for search_dir in search_dirs:
            for ext in extensions:
                for file_path in search_dir.rglob(f"*{ext}"):
                    if self._should_include_file(file_path):
                        files.append(file_path)

        return list(set(files))

    def _should_include_file(self, file_path: Path) -> bool:
        """Check if file should be processed"""
        file_str = str(file_path)

        # Exclude patterns
        if any(exclude in file_str for exclude in self.exclude_dirs):
            return False

        # Include patterns
        if self.include_dirs:
            return any(file_path.is_relative_to(self.project_root / include_dir)
                       for include_dir in self.include_dirs
                       if (self.project_root / include_dir).exists())

        return True

    def _get_file_hash(self, file_path: Path) -> str:
        """Get file hash for change detection"""
        try:
            with open(file_path, 'rb') as f:
                return hashlib.md5(f.read()).hexdigest()
        except Exception:
            return ""

    def _get_git_commit(self) -> Optional[str]:
        """Get current git commit hash"""
        try:
            result = subprocess.run(
                ["git", "rev-parse", "HEAD"],
                cwd=self.project_root,
                capture_output=True,
                text=True,
                timeout=10
            )
            return result.stdout.strip() if result.returncode == 0 else None
        except Exception:
            return None

    def _get_git_changes(self) -> List[str]:
        """Get changed files from git"""
        try:
            result = subprocess.run(
                ["git", "diff", "--name-only", "HEAD~1", "HEAD"],
                cwd=self.project_root,
                capture_output=True,
                text=True,
                timeout=10
            )
            return result.stdout.strip().split('\n') if result.returncode == 0 else []
        except Exception:
            return []

    def _extract_code_elements(self, file_path: Path) -> List[CodeElement]:
        """Extract code elements from Python file"""
        elements = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            tree = ast.parse(content)

            for node in ast.walk(tree):
                if isinstance(node, ast.ClassDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='class',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=f"class {node.name}",
                        docstring=ast.get_docstring(node),
                        hash_signature=hashlib.md5(f"class {node.name}".encode()).hexdigest()
                    ))

                    # Add methods
                    for item in node.body:
                        if isinstance(item, ast.FunctionDef):
                            elements.append(CodeElement(
                                name=item.name,
                                element_type='method',
                                file_path=str(file_path),
                                line_start=item.lineno,
                                line_end=getattr(item, 'end_lineno', item.lineno),
                                signature=f"def {item.name}",
                                docstring=ast.get_docstring(item),
                                parent_class=node.name,
                                hash_signature=hashlib.md5(f"def {item.name}".encode()).hexdigest()
                            ))

                elif isinstance(node, ast.FunctionDef):
                    elements.append(CodeElement(
                        name=node.name,
                        element_type='function',
                        file_path=str(file_path),
                        line_start=node.lineno,
                        line_end=getattr(node, 'end_lineno', node.lineno),
                        signature=f"def {node.name}",
                        docstring=ast.get_docstring(node),
                        hash_signature=hashlib.md5(f"def {node.name}".encode()).hexdigest()
                    ))

        except Exception as e:
            logger.error(f"Error extracting elements from {file_path}: {e}")

        return elements

    def _parse_markdown_file(self, file_path: Path) -> List[DocSection]:
        """Enhanced markdown file parsing with better error handling"""
        sections = []

        try:
            # Skip non-markdown files and hidden files
            if not file_path.suffix.lower() == '.md' or file_path.name.startswith('.'):
                return sections

            with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
                content = f.read()

            if not content.strip():
                return sections

            lines = content.split('\n')
            current_section = None
            section_content = []
            line_start = 0

            for i, line in enumerate(lines):
                # Look for markdown headers with improved regex
                header_match = re.match(r'^(#{1,6})\s+(.+)$', line.strip())

                if header_match:
                    # Save previous section if exists
                    if current_section:
                        section = self._create_doc_section(
                            file_path, current_section, section_content, line_start, i - 1
                        )
                        if section:  # Only add non-empty sections
                            sections.append(section)

                    # Start new section
                    level = len(header_match.group(1))
                    title = header_match.group(2).strip()

                    # Skip empty titles
                    if not title:
                        continue

                    current_section = (title, level)
                    section_content = []
                    line_start = i

                elif current_section:
                    section_content.append(line)

            # Save last section
            if current_section:
                section = self._create_doc_section(
                    file_path, current_section, section_content, line_start, len(lines) - 1
                )
                if section:
                    sections.append(section)

            if sections:
                logger.debug(f"Parsed {len(sections)} sections from {file_path}")

        except Exception as e:
            logger.error(f"Error parsing markdown {file_path}: {e}")

        return sections

    def _create_doc_section(self, file_path: Path, section_info: Tuple[str, int],
                            content_lines: List[str], line_start: int, line_end: int) -> Optional[DocSection]:
        """Create DocSection with validation"""
        try:
            title, level = section_info
            content = '\n'.join(content_lines).strip()

            # Skip sections with no meaningful content
            if len(content) < 10 and not any(c.isalnum() for c in content):
                return None

            # Extract tags and references with safer regex
            tags = []
            source_refs = []

            try:
                tags = re.findall(r'#([a-zA-Z][a-zA-Z0-9_-]*)', content)
                source_refs = re.findall(r'`([^`]+\.py:[^`]+)`', content)
            except Exception:
                pass  # If regex fails, continue with empty lists

            section_id = f"{file_path.name}#{title}"
            content_hash = hashlib.md5(content.encode()).hexdigest()
            combined_hash = hashlib.md5(f"{title}:{content}:{line_start}".encode()).hexdigest()

            return DocSection(
                section_id=section_id,
                file_path=str(file_path),
                title=title,
                content=content,
                level=level,
                line_start=line_start,
                line_end=line_end,
                source_refs=source_refs,
                tags=tags,
                hash_signature=combined_hash,
                content_hash=content_hash,
                last_modified=datetime.fromtimestamp(
                    file_path.stat().st_mtime) if file_path.exists() else datetime.now()
            )
        except Exception as e:
            logger.error(f"Error creating doc section: {e}")
            return None

    def _remove_file_from_index(self, file_path: Path):
        """Remove all references to a file from index"""
        file_str = str(file_path)

        # Remove sections
        to_remove = [k for k, v in self.current_index.sections.items() if v.file_path == file_str]
        for key in to_remove:
            del self.current_index.sections[key]

        # Remove code elements
        to_remove = [k for k, v in self.current_index.code_elements.items() if v.file_path == file_str]
        for key in to_remove:
            del self.current_index.code_elements[key]

        # Remove file hash
        if file_str in self.current_index.file_hashes:
            del self.current_index.file_hashes[file_str]

    def _update_file_in_index(self, file_path: Path):
        """Update index for a specific file"""
        # Remove old entries
        self._remove_file_from_index(file_path)

        # Add new entries
        try:
            if file_path.suffix == '.py':
                elements = self._extract_code_elements(file_path)
                for element in elements:
                    element_id = f"{element.file_path}:{element.name}"
                    if element.parent_class:
                        element_id = f"{element.file_path}:{element.parent_class}.{element.name}"
                    self.current_index.code_elements[element_id] = element

            elif file_path.suffix == '.md' and file_path.is_relative_to(self.docs_root):
                sections = self._parse_markdown_file(file_path)
                for section in sections:
                    self.current_index.sections[section.section_id] = section

        except Exception as e:
            logger.error(f"Error updating file {file_path}: {e}")


    def source_code_lookup(self,
                           element_name: Optional[str] = None,
                           file_path: Optional[str] = None,
                           element_type: Optional[str] = None,
                           max_results: int = 25,
                           return_code_block: bool = True) -> Result:
        """Look up source code elements with option to return single method code blocks"""
        try:
            if not self.current_index:
                self.current_index = self._load_index(minimal=False)

            matches = []

            for element_id, element in self.current_index.code_elements.items():
                if len(matches) >= max_results:
                    break

                # Apply filters
                if element_name and element_name.lower() not in element.name.lower():
                    continue
                if file_path and file_path not in element.file_path:
                    continue
                if element_type and element.element_type != element_type:
                    continue

                # Get code block for this specific element
                code_block = ""
                if return_code_block:
                    code_block = self._extract_single_code_block(element)

                # Find related docs
                related_docs = []
                for section_id, section in self.current_index.sections.items():
                    if len(related_docs) >= 3:  # Limit related docs
                        break
                    if (element_id in section.source_refs or
                        element.name in section.content[:200] or
                        element.name in section.title):
                        related_docs.append({
                            "section_id": section_id,
                            "title": section.title,
                            "file_path": section.file_path
                        })

                match_data = {
                    "element_id": element_id,
                    "name": element.name,
                    "type": element.element_type,
                    "signature": element.signature,
                    "file_path": element.file_path,
                    "line_start": element.line_start,
                    "line_end": element.line_end,
                    "parent_class": element.parent_class,
                    "docstring": element.docstring[:300] if element.docstring else None,
                    "related_documentation": related_docs
                }

                if return_code_block and code_block:
                    match_data["code_block"] = code_block

                matches.append(match_data)

            return Result.ok({
                "matches": matches,
                "total_matches": len(matches),
                "total_available": len(self.current_index.code_elements),
                "truncated": len(matches) >= max_results
            })

        except Exception as e:
            logger.error(f"Error in source code lookup: {e}")
            return Result.default_user_error(f"Error looking up source code: {e}")

    def _extract_single_code_block(self, element: CodeElement) -> str:
        """Extract single method/class code block from source file"""
        try:
            with open(element.file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Get lines for this element only
            start_line = max(0, element.line_start - 1)  # Convert to 0-based
            end_line = min(len(lines), element.line_end)

            element_lines = lines[start_line:end_line]

            # Clean up the code block
            if element_lines:
                # Remove common leading whitespace
                import textwrap
                code_block = textwrap.dedent(''.join(element_lines))

                # Add language hint for syntax highlighting
                return f"```python\n{code_block}```"

            return ""

        except Exception as e:
            logger.error(f"Error extracting code block for {element.name}: {e}")
            return ""

    # Add the missing method to the app registration
    def add_source_code_lookup_to_app(self):
        """Add source code lookup method to app"""
        if hasattr(self, 'app'):
            self.app.source_code_lookup = self.source_code_lookup
add_source_code_lookup_to_app()

Add source code lookup method to app

Source code in toolboxv2/utils/extras/mkdocs.py
2500
2501
2502
2503
def add_source_code_lookup_to_app(self):
    """Add source code lookup method to app"""
    if hasattr(self, 'app'):
        self.app.source_code_lookup = self.source_code_lookup
auto_update_docs(dry_run=False, max_updates=5, timeout=30) async

Automatically update documentation with timeout protection

Source code in toolboxv2/utils/extras/mkdocs.py
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
async def auto_update_docs(self, dry_run: bool = False, max_updates: int = 5,
                           timeout: int = 30) -> Result:
    """Automatically update documentation with timeout protection"""
    try:
        # Get suggestions
        suggestions_result = await asyncio.wait_for(
            self.get_update_suggestions(max_suggestions=max_updates * 2),
            timeout=10.0
        )

        if suggestions_result.is_error():
            return suggestions_result

        suggestions = suggestions_result.get("suggestions")[:max_updates]

        if dry_run:
            return Result.ok({
                "dry_run": True,
                "would_update": len(suggestions),
                "suggestions": [s["type"] + ": " + s.get("element_name", s.get("title", ""))
                                for s in suggestions]
            })

        results = []
        start_time = time.time()

        for suggestion in suggestions:
            if time.time() - start_time > timeout:
                break

            try:
                if suggestion["action"] == "generate_from_code":
                    result = await asyncio.wait_for(
                        self.docs_writer(
                            action="generate_from_code",
                            source_file=suggestion["file_path"],
                            auto_generate=True
                        ), timeout=10.0
                    )
                elif suggestion["action"] == "add_section":
                    result = await asyncio.wait_for(
                        self.docs_writer(
                            action="add_section",
                            file_path=f"{Path(suggestion['file_path']).stem}.md",
                            section_title=suggestion["element_name"],
                            source_file=suggestion["file_path"],
                            auto_generate=True
                        ), timeout=8.0
                    )
                else:
                    continue

                results.append({
                    "suggestion": suggestion["type"],
                    "status": "success" if result.is_ok() else "error",
                    "details": str(result.error) if result.is_error() else "completed"
                })

            except asyncio.TimeoutError:
                results.append({
                    "suggestion": suggestion["type"],
                    "status": "timeout",
                    "details": "Operation timed out"
                })

            await asyncio.sleep(0.1)  # Brief pause between operations

        return Result.ok({
            "processed": len(results),
            "successful": len([r for r in results if r["status"] == "success"]),
            "results": results,
            "execution_time": f"{time.time() - start_time:.2f}s"
        })

    except Exception as e:
        logger.error(f"Error in auto_update_docs: {e}")
        return Result.default_user_error(f"Auto-update failed: {e}")
docs_reader(query=None, section_id=None, file_path=None, tags=None, include_source_refs=True, format_type='structured', max_results=25) async

Ultra-fast unified docs reader with proper index loading

Source code in toolboxv2/utils/extras/mkdocs.py
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
async def docs_reader(self,
                      query: Optional[str] = None,
                      section_id: Optional[str] = None,
                      file_path: Optional[str] = None,
                      tags: Optional[List[str]] = None,
                      include_source_refs: bool = True,
                      format_type: str = "structured",
                      max_results: int = 25) -> Result:
    """Ultra-fast unified docs reader with proper index loading"""
    try:
        start_time = time.time()

        # Load index if needed - try cached first, then saved, then build
        if not self.current_index:
            # First try to load from saved file
            self.current_index = self._load_index(minimal=False)

            # If no sections found, suggest running initial_docs_parse
            if not self.current_index.sections:
                if self.index_file.exists():
                    logger.warning("Index file exists but contains no documentation sections")
                    # Try to find any markdown files
                    md_files = list(self.docs_root.rglob('*.md'))
                    if md_files:
                        logger.info(f"Found {len(md_files)} markdown files, re-parsing...")
                        # Re-parse markdown files
                        for md_file in md_files[:10]:  # Limit for quick check
                            sections = self._parse_markdown_file(md_file)
                            for section in sections:
                                self.current_index.sections[section.section_id] = section

                        if self.current_index.sections:
                            logger.info(f"Re-parsed {len(self.current_index.sections)} sections")
                            await self._save_index_async(self.current_index)

                if not self.current_index.sections:
                    return Result.default_user_error(
                        "No documentation sections found. Run initial_docs_parse() to build the documentation index, "
                        "or create some .md files in the docs directory."
                    )

        # Check cache for repeated queries
        cache_key = f"{query}:{section_id}:{file_path}:{tags}:{format_type}:{max_results}"
        if cache_key in self._search_cache:
            cache_entry = self._search_cache[cache_key]
            if time.time() - cache_entry['timestamp'] < self._cache_timeout:
                return Result.ok(cache_entry['result'])

        # Fast path for specific section
        if section_id:
            if section_id in self.current_index.sections:
                section = self.current_index.sections[section_id]
                result = self._format_single_section(section, include_source_refs, format_type)
                return result
            else:
                return Result.default_user_error(f"Section not found: {section_id}")

        # Search and format results
        matching_sections = await self._search_sections(query, file_path, tags, max_results, start_time)
        result_data = self._format_sections(matching_sections, include_source_refs, format_type)

        # Cache the result
        self._search_cache[cache_key] = {
            'result': result_data,
            'timestamp': time.time()
        }

        return Result.ok(result_data)

    except Exception as e:
        logger.error(f"Error in docs_reader: {e}")
        return Result.default_user_error(f"Error reading documentation: {e}")
docs_writer(action, file_path=None, section_title=None, content=None, source_file=None, auto_generate=False, position=None, level=2) async

Unified optimized docs writer

Source code in toolboxv2/utils/extras/mkdocs.py
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
async def docs_writer(self,
                      action: str,
                      file_path: Optional[str] = None,
                      section_title: Optional[str] = None,
                      content: Optional[str] = None,
                      source_file: Optional[str] = None,
                      auto_generate: bool = False,
                      position: Optional[str] = None,
                      level: int = 2) -> Result:
    """Unified optimized docs writer"""
    try:
        if not self.current_index:
            self.current_index = self._load_index(minimal=True)

        result = {"action": action, "timestamp": datetime.now().isoformat()}

        if action == "update_section":
            return await self._update_section(file_path, section_title, content,
                                              source_file, auto_generate)
        elif action == "add_section":
            return await self._add_section(file_path, section_title, content,
                                           source_file, auto_generate, position, level)
        elif action == "create_file":
            return await self._create_file(file_path, content, source_file, auto_generate)
        elif action == "generate_from_code":
            return await self._generate_from_code(source_file, file_path, auto_generate)
        else:
            return Result.default_user_error(f"Unknown action: {action}")

    except Exception as e:
        logger.error(f"Error in docs_writer: {e}")
        return Result.default_user_error(f"Error writing documentation: {e}")
get_update_suggestions(force_scan=False, max_suggestions=50) async

Get prioritized documentation update suggestions

Source code in toolboxv2/utils/extras/mkdocs.py
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
async def get_update_suggestions(self, force_scan: bool = False,
                                 max_suggestions: int = 50) -> Result:
    """Get prioritized documentation update suggestions"""
    try:
        if not self.current_index:
            self.current_index = self._load_index()

        # Quick scan for changes if needed
        if force_scan:
            await self._quick_update_index()

        suggestions = []

        # Find undocumented code elements
        undocumented = self._find_undocumented_elements()
        for element in undocumented[:max_suggestions // 2]:
            priority = self._assess_priority(element)
            suggestions.append({
                "type": "missing_documentation",
                "element_name": element.name,
                "element_type": element.element_type,
                "file_path": element.file_path,
                "priority": priority,
                "action": "generate_from_code" if element.element_type == "class" else "add_section"
            })

        # Find unclear sections
        unclear = self._find_unclear_sections()
        for section_id in unclear[:max_suggestions // 2]:
            section = self.current_index.sections[section_id]
            suggestions.append({
                "type": "unclear_documentation",
                "section_id": section_id,
                "title": section.title,
                "file_path": section.file_path,
                "priority": "medium",
                "action": "update_section"
            })

        # Sort by priority
        priority_order = {"high": 0, "medium": 1, "low": 2}
        suggestions.sort(key=lambda x: priority_order.get(x["priority"], 3))

        return Result.ok({
            "suggestions": suggestions[:max_suggestions],
            "total_found": len(suggestions),
            "undocumented_elements": len(undocumented),
            "unclear_sections": len(unclear)
        })

    except Exception as e:
        logger.error(f"Error getting update suggestions: {e}")
        return Result.default_user_error(f"Error analyzing updates: {e}")
initial_docs_parse(update_index=True, force_rebuild=False) async

Parse existing documentation and build initial index with proper saving

Source code in toolboxv2/utils/extras/mkdocs.py
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
async def initial_docs_parse(self, update_index: bool = True, force_rebuild: bool = False) -> Result:
    """Parse existing documentation and build initial index with proper saving"""
    try:
        logger.info("Starting initial documentation parse...")

        # Check if we can use existing index
        if not force_rebuild and self.index_file.exists() and not update_index:
            self.current_index = self._load_index(minimal=False)
            if self.current_index.sections or self.current_index.code_elements:
                logger.info("Using existing index")
                return Result.ok({
                    "total_sections": len(self.current_index.sections),
                    "total_code_elements": len(self.current_index.code_elements),
                    "linked_sections": len([s for s in self.current_index.sections.values() if s.source_refs]),
                    "completion_rate": f"{(len([s for s in self.current_index.sections.values() if s.source_refs]) / max(len(self.current_index.sections), 1) * 100):.1f}%",
                    "used_cached": True
                })

        if update_index or force_rebuild:
            # Build comprehensive index
            logger.info("Building comprehensive index from source files...")
            self.current_index = await asyncio.get_event_loop().run_in_executor(
                None, self._build_full_index
            )
        else:
            self.current_index = self._load_index(minimal=False)

        # Ensure we have some documentation sections
        if not self.current_index.sections:
            logger.info("No documentation sections found, scanning for markdown files...")
            # Look for markdown files in docs and project root
            search_paths = [self.docs_root]
            if self.docs_root != self.project_root:
                search_paths.append(self.project_root)

            for search_path in search_paths:
                for md_file in search_path.rglob('*.md'):
                    if self._should_include_file(md_file):
                        sections = self._parse_markdown_file(md_file)
                        for section in sections:
                            self.current_index.sections[section.section_id] = section

                # Don't search too deep
                if len(self.current_index.sections) > 0:
                    break

            logger.info(f"Found {len(self.current_index.sections)} documentation sections")

        # Link documentation sections to code elements
        linked_count = await self._link_docs_to_code()

        # Save updated index with proper error handling
        try:
            await self._save_index_async(self.current_index)
            logger.info("Index saved successfully")
        except Exception as e:
            logger.error(f"Failed to save index: {e}")
            # Continue anyway, we have the index in memory

        completion_rate = (
                linked_count / max(len(self.current_index.sections), 1) * 100) if self.current_index.sections else 0

        return Result.ok({
            "total_sections": len(self.current_index.sections),
            "total_code_elements": len(self.current_index.code_elements),
            "linked_sections": linked_count,
            "completion_rate": f"{completion_rate:.1f}%",
            "index_file": str(self.index_file),
            "docs_root": str(self.docs_root)
        })

    except Exception as e:
        logger.error(f"Error in initial_docs_parse: {e}")
        return Result.default_user_error(f"Failed to parse docs: {e}")
source_code_lookup(element_name=None, file_path=None, element_type=None, max_results=25, return_code_block=True)

Look up source code elements with option to return single method code blocks

Source code in toolboxv2/utils/extras/mkdocs.py
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
def source_code_lookup(self,
                       element_name: Optional[str] = None,
                       file_path: Optional[str] = None,
                       element_type: Optional[str] = None,
                       max_results: int = 25,
                       return_code_block: bool = True) -> Result:
    """Look up source code elements with option to return single method code blocks"""
    try:
        if not self.current_index:
            self.current_index = self._load_index(minimal=False)

        matches = []

        for element_id, element in self.current_index.code_elements.items():
            if len(matches) >= max_results:
                break

            # Apply filters
            if element_name and element_name.lower() not in element.name.lower():
                continue
            if file_path and file_path not in element.file_path:
                continue
            if element_type and element.element_type != element_type:
                continue

            # Get code block for this specific element
            code_block = ""
            if return_code_block:
                code_block = self._extract_single_code_block(element)

            # Find related docs
            related_docs = []
            for section_id, section in self.current_index.sections.items():
                if len(related_docs) >= 3:  # Limit related docs
                    break
                if (element_id in section.source_refs or
                    element.name in section.content[:200] or
                    element.name in section.title):
                    related_docs.append({
                        "section_id": section_id,
                        "title": section.title,
                        "file_path": section.file_path
                    })

            match_data = {
                "element_id": element_id,
                "name": element.name,
                "type": element.element_type,
                "signature": element.signature,
                "file_path": element.file_path,
                "line_start": element.line_start,
                "line_end": element.line_end,
                "parent_class": element.parent_class,
                "docstring": element.docstring[:300] if element.docstring else None,
                "related_documentation": related_docs
            }

            if return_code_block and code_block:
                match_data["code_block"] = code_block

            matches.append(match_data)

        return Result.ok({
            "matches": matches,
            "total_matches": len(matches),
            "total_available": len(self.current_index.code_elements),
            "truncated": len(matches) >= max_results
        })

    except Exception as e:
        logger.error(f"Error in source code lookup: {e}")
        return Result.default_user_error(f"Error looking up source code: {e}")
MarkdownParser

Parses markdown files and extracts sections

Source code in toolboxv2/utils/extras/mkdocs.py
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
class MarkdownParser:
    """Parses markdown files and extracts sections"""

    def parse_file(self, file_path: Path) -> List[DocSection]:
        """Parse markdown file into sections"""
        sections = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            lines = content.split('\n')
            current_section = None
            section_content = []
            line_start = 0

            for i, line in enumerate(lines):
                # Check for header
                header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

                if header_match:
                    # Save previous section
                    if current_section:
                        sections.append(self._create_section(
                            file_path, current_section, section_content,
                            line_start, i - 1
                        ))

                    # Start new section
                    level = len(header_match.group(1))
                    title = header_match.group(2).strip()
                    current_section = (title, level)
                    section_content = []
                    line_start = i

                elif current_section:
                    section_content.append(line)

            # Save last section
            if current_section:
                sections.append(self._create_section(
                    file_path, current_section, section_content,
                    line_start, len(lines) - 1
                ))

        except Exception as e:
            logger.error(f"Error parsing markdown file {file_path}: {e}")

        return sections

    def _create_section(self, file_path: Path, section_info: Tuple[str, int],
                        content_lines: List[str], line_start: int, line_end: int) -> DocSection:
        """Create DocSection from parsed data"""
        title, level = section_info
        content = '\n'.join(content_lines).strip()

        # Extract source references from content
        source_refs = self._extract_source_refs(content)

        # Extract tags
        tags = self._extract_tags(content)

        section_id = f"{file_path.name}#{title}"

        return DocSection(
            section_id=section_id,
            file_path=str(file_path),
            title=title,
            content=content,
            level=level,
            line_start=line_start,
            line_end=line_end,
            source_refs=source_refs,
            tags=tags,
            hash_signature=hashlib.md5(content.encode()).hexdigest(),
            last_modified=datetime.fromtimestamp(file_path.stat().st_mtime)
        )

    def _extract_source_refs(self, content: str) -> List[str]:
        """Extract source code references from markdown content"""
        refs = []

        # Look for code references in various formats
        patterns = [
            r'`([^`]+\.py:[^`]+)`',  # `file.py:Class.method`
            r'\[([^\]]+)\]\([^)]*\.py[^)]*\)',  # [text](file.py)
            r'```python[^`]*?([a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*)[^`]*?```'  # code blocks
        ]

        for pattern in patterns:
            matches = re.findall(pattern, content)
            refs.extend(matches)

        return list(set(refs))  # Remove duplicates

    def _extract_tags(self, content: str) -> List[str]:
        """Extract tags from markdown content"""
        tags = []

        # Look for tags in various formats
        tag_patterns = [
            r'Tags?:\s*([^\n]+)',  # Tags: tag1, tag2
            r'#([a-zA-Z][a-zA-Z0-9_-]*)',  # #hashtag
        ]

        for pattern in tag_patterns:
            matches = re.findall(pattern, content, re.IGNORECASE)
            for match in matches:
                if ',' in match:
                    tags.extend([tag.strip() for tag in match.split(',')])
                else:
                    tags.append(match.strip())

        return list(set(tags))

    def parse_file_incremental(self, file_path: Path, existing_sections: Dict[str, DocSection] = None) -> Tuple[
        List[DocSection], List[str]]:
        """Parse file with section-level change detection"""
        if existing_sections is None:
            existing_sections = {}

        new_sections = []
        changed_sections = []

        try:
            with open(file_path, 'r', encoding='utf-8') as f:
                content = f.read()

            lines = content.split('\n')
            current_section = None
            section_content = []
            line_start = 0

            for i, line in enumerate(lines):
                header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

                if header_match:
                    # Process previous section
                    if current_section:
                        section = self._create_section_with_hash(
                            file_path, current_section, section_content,
                            line_start, i - 1
                        )

                        # Check if section changed
                        existing_section = existing_sections.get(section.section_id)
                        if self._section_changed(section, existing_section):
                            section.change_detected = True
                            changed_sections.append(section.section_id)

                        new_sections.append(section)

                    # Start new section
                    level = len(header_match.group(1))
                    title = header_match.group(2).strip()
                    current_section = (title, level)
                    section_content = []
                    line_start = i

                elif current_section:
                    section_content.append(line)

            # Handle last section
            if current_section:
                section = self._create_section_with_hash(
                    file_path, current_section, section_content,
                    line_start, len(lines) - 1
                )

                existing_section = existing_sections.get(section.section_id)
                if self._section_changed(section, existing_section):
                    section.change_detected = True
                    changed_sections.append(section.section_id)

                new_sections.append(section)

        except Exception as e:
            logger.error(f"Error parsing markdown file {file_path}: {e}")

        return new_sections, changed_sections

    def _create_section_with_hash(self, file_path: Path, section_info: Tuple[str, int],
                                  content_lines: List[str], line_start: int, line_end: int) -> DocSection:
        """Create DocSection with precise hash tracking"""
        title, level = section_info
        content = '\n'.join(content_lines).strip()

        # Create separate hashes for different aspects
        content_hash = hashlib.md5(content.encode()).hexdigest()
        title_hash = hashlib.md5(title.encode()).hexdigest()
        combined_hash = hashlib.md5(f"{title}:{content}:{line_start}:{line_end}".encode()).hexdigest()

        source_refs = self._extract_source_refs(content)
        tags = self._extract_tags(content)
        section_id = f"{file_path.name}#{title}"

        return DocSection(
            section_id=section_id,
            file_path=str(file_path),
            title=title,
            content=content,
            level=level,
            line_start=line_start,
            line_end=line_end,
            source_refs=source_refs,
            tags=tags,
            hash_signature=combined_hash,
            content_hash=content_hash,
            last_modified=datetime.fromtimestamp(file_path.stat().st_mtime),
            change_detected=False
        )

    def _section_changed(self, new_section: DocSection, existing_section: Optional[DocSection]) -> bool:
        """Check if section actually changed"""
        if not existing_section:
            return True  # New section

        # Quick hash comparison
        if new_section.content_hash == existing_section.content_hash:
            return False

        # Title change
        if new_section.title != existing_section.title:
            return True

        # Significant content change (not just whitespace)
        if new_section.hash_signature != existing_section.hash_signature:
            return True

        return False
parse_file(file_path)

Parse markdown file into sections

Source code in toolboxv2/utils/extras/mkdocs.py
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
def parse_file(self, file_path: Path) -> List[DocSection]:
    """Parse markdown file into sections"""
    sections = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        lines = content.split('\n')
        current_section = None
        section_content = []
        line_start = 0

        for i, line in enumerate(lines):
            # Check for header
            header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

            if header_match:
                # Save previous section
                if current_section:
                    sections.append(self._create_section(
                        file_path, current_section, section_content,
                        line_start, i - 1
                    ))

                # Start new section
                level = len(header_match.group(1))
                title = header_match.group(2).strip()
                current_section = (title, level)
                section_content = []
                line_start = i

            elif current_section:
                section_content.append(line)

        # Save last section
        if current_section:
            sections.append(self._create_section(
                file_path, current_section, section_content,
                line_start, len(lines) - 1
            ))

    except Exception as e:
        logger.error(f"Error parsing markdown file {file_path}: {e}")

    return sections
parse_file_incremental(file_path, existing_sections=None)

Parse file with section-level change detection

Source code in toolboxv2/utils/extras/mkdocs.py
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
def parse_file_incremental(self, file_path: Path, existing_sections: Dict[str, DocSection] = None) -> Tuple[
    List[DocSection], List[str]]:
    """Parse file with section-level change detection"""
    if existing_sections is None:
        existing_sections = {}

    new_sections = []
    changed_sections = []

    try:
        with open(file_path, 'r', encoding='utf-8') as f:
            content = f.read()

        lines = content.split('\n')
        current_section = None
        section_content = []
        line_start = 0

        for i, line in enumerate(lines):
            header_match = re.match(r'^(#{1,6})\s+(.+)$', line)

            if header_match:
                # Process previous section
                if current_section:
                    section = self._create_section_with_hash(
                        file_path, current_section, section_content,
                        line_start, i - 1
                    )

                    # Check if section changed
                    existing_section = existing_sections.get(section.section_id)
                    if self._section_changed(section, existing_section):
                        section.change_detected = True
                        changed_sections.append(section.section_id)

                    new_sections.append(section)

                # Start new section
                level = len(header_match.group(1))
                title = header_match.group(2).strip()
                current_section = (title, level)
                section_content = []
                line_start = i

            elif current_section:
                section_content.append(line)

        # Handle last section
        if current_section:
            section = self._create_section_with_hash(
                file_path, current_section, section_content,
                line_start, len(lines) - 1
            )

            existing_section = existing_sections.get(section.section_id)
            if self._section_changed(section, existing_section):
                section.change_detected = True
                changed_sections.append(section.section_id)

            new_sections.append(section)

    except Exception as e:
        logger.error(f"Error parsing markdown file {file_path}: {e}")

    return new_sections, changed_sections
SectionManager

Manages precise operations on documentation sections

Source code in toolboxv2/utils/extras/mkdocs.py
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
class SectionManager:
    """Manages precise operations on documentation sections"""

    def __init__(self, docs_root: Path):
        self.docs_root = docs_root

    def create_new_file(self, file_path: str, initial_content: str = "") -> Result:
        """Create a new markdown file"""
        try:
            full_path = self.docs_root / file_path
            full_path.parent.mkdir(parents=True, exist_ok=True)

            if full_path.exists():
                return Result.default_user_error(f"File already exists: {file_path}")

            with open(full_path, 'w', encoding='utf-8') as f:
                f.write(initial_content)

            return Result.ok({"file_path": str(full_path), "action": "created"})

        except Exception as e:
            return Result.default_user_error(f"Error creating file: {e}")

    def add_section(self, file_path: str, section: DocSection, position: Optional[str] = None) -> Result:
        """Add a new section to a markdown file"""
        try:
            full_path = self.docs_root / file_path

            if not full_path.exists():
                # Create new file
                content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
                with open(full_path, 'w', encoding='utf-8') as f:
                    f.write(content)
                return Result.ok({"action": "created_file_with_section"})

            # Read existing content
            with open(full_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Find insertion point
            insert_index = len(lines)  # Default: append at end

            if position:
                if position == "top":
                    insert_index = 0
                elif position.startswith("after:"):
                    target_section = position[6:]
                    for i, line in enumerate(lines):
                        if line.strip().endswith(target_section):
                            # Find end of this section
                            for j in range(i + 1, len(lines)):
                                if re.match(r'^#{1,6}\s+', lines[j]):
                                    insert_index = j
                                    break
                            break

            # Insert new section
            new_content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
            lines.insert(insert_index, new_content)

            # Write back
            with open(full_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return Result.ok({"action": "section_added", "position": insert_index})

        except Exception as e:
            return Result.default_user_error(f"Error adding section: {e}")

    def update_section(self, section: DocSection, new_content: str) -> Result:
        """Update an existing section"""
        try:
            file_path = Path(section.file_path)

            with open(file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Replace content between line_start and line_end
            new_section_content = f"{'#' * section.level} {section.title}\n\n{new_content}\n\n"
            new_lines = new_section_content.split('\n')

            # Replace the section
            lines[section.line_start:section.line_end + 1] = [line + '\n' for line in new_lines]

            # Write back
            with open(file_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return Result.ok({"action": "section_updated"})

        except Exception as e:
            return Result.default_user_error(f"Error updating section: {e}")

    def delete_section(self, section: DocSection) -> Result:
        """Delete a section from a file"""
        try:
            file_path = Path(section.file_path)

            with open(file_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()

            # Remove lines for this section
            del lines[section.line_start:section.line_end + 1]

            # Write back
            with open(file_path, 'w', encoding='utf-8') as f:
                f.writelines(lines)

            return Result.ok({"action": "section_deleted"})

        except Exception as e:
            return Result.default_user_error(f"Error deleting section: {e}")
add_section(file_path, section, position=None)

Add a new section to a markdown file

Source code in toolboxv2/utils/extras/mkdocs.py
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
def add_section(self, file_path: str, section: DocSection, position: Optional[str] = None) -> Result:
    """Add a new section to a markdown file"""
    try:
        full_path = self.docs_root / file_path

        if not full_path.exists():
            # Create new file
            content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
            with open(full_path, 'w', encoding='utf-8') as f:
                f.write(content)
            return Result.ok({"action": "created_file_with_section"})

        # Read existing content
        with open(full_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        # Find insertion point
        insert_index = len(lines)  # Default: append at end

        if position:
            if position == "top":
                insert_index = 0
            elif position.startswith("after:"):
                target_section = position[6:]
                for i, line in enumerate(lines):
                    if line.strip().endswith(target_section):
                        # Find end of this section
                        for j in range(i + 1, len(lines)):
                            if re.match(r'^#{1,6}\s+', lines[j]):
                                insert_index = j
                                break
                        break

        # Insert new section
        new_content = f"{'#' * section.level} {section.title}\n\n{section.content}\n\n"
        lines.insert(insert_index, new_content)

        # Write back
        with open(full_path, 'w', encoding='utf-8') as f:
            f.writelines(lines)

        return Result.ok({"action": "section_added", "position": insert_index})

    except Exception as e:
        return Result.default_user_error(f"Error adding section: {e}")
create_new_file(file_path, initial_content='')

Create a new markdown file

Source code in toolboxv2/utils/extras/mkdocs.py
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
def create_new_file(self, file_path: str, initial_content: str = "") -> Result:
    """Create a new markdown file"""
    try:
        full_path = self.docs_root / file_path
        full_path.parent.mkdir(parents=True, exist_ok=True)

        if full_path.exists():
            return Result.default_user_error(f"File already exists: {file_path}")

        with open(full_path, 'w', encoding='utf-8') as f:
            f.write(initial_content)

        return Result.ok({"file_path": str(full_path), "action": "created"})

    except Exception as e:
        return Result.default_user_error(f"Error creating file: {e}")
delete_section(section)

Delete a section from a file

Source code in toolboxv2/utils/extras/mkdocs.py
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
def delete_section(self, section: DocSection) -> Result:
    """Delete a section from a file"""
    try:
        file_path = Path(section.file_path)

        with open(file_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        # Remove lines for this section
        del lines[section.line_start:section.line_end + 1]

        # Write back
        with open(file_path, 'w', encoding='utf-8') as f:
            f.writelines(lines)

        return Result.ok({"action": "section_deleted"})

    except Exception as e:
        return Result.default_user_error(f"Error deleting section: {e}")
update_section(section, new_content)

Update an existing section

Source code in toolboxv2/utils/extras/mkdocs.py
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
def update_section(self, section: DocSection, new_content: str) -> Result:
    """Update an existing section"""
    try:
        file_path = Path(section.file_path)

        with open(file_path, 'r', encoding='utf-8') as f:
            lines = f.readlines()

        # Replace content between line_start and line_end
        new_section_content = f"{'#' * section.level} {section.title}\n\n{new_content}\n\n"
        new_lines = new_section_content.split('\n')

        # Replace the section
        lines[section.line_start:section.line_end + 1] = [line + '\n' for line in new_lines]

        # Write back
        with open(file_path, 'w', encoding='utf-8') as f:
            f.writelines(lines)

        return Result.ok({"action": "section_updated"})

    except Exception as e:
        return Result.default_user_error(f"Error updating section: {e}")
TOCEntry

Represents a table of contents entry

Source code in toolboxv2/utils/extras/mkdocs.py
 95
 96
 97
 98
 99
100
101
102
103
104
105
class TOCEntry:
    """Represents a table of contents entry"""

    def __init__(self, title: str, file_path: str, level: int, line_number: int):
        self.title = title
        self.file_path = file_path
        self.level = level
        self.line_number = line_number
        self.has_implementation = False
        self.is_unclear = False
        self.source_refs: List[str] = []
add_to_app(app, include_dirs=None, exclude_dirs=None)

Add optimized markdown docs system to app

Source code in toolboxv2/utils/extras/mkdocs.py
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
def add_to_app(app: AppType, include_dirs: List[str] = None, exclude_dirs: List[str] = None) -> MarkdownDocsSystem:
    """Add optimized markdown docs system to app"""
    from toolboxv2 import tb_root_dir

    docs_system = MarkdownDocsSystem(
        app,
        docs_root=str(tb_root_dir.parent / "docs"),
        include_dirs=include_dirs,
        exclude_dirs=exclude_dirs
    )

    # Register core functions
    app.docs_reader = docs_system.docs_reader
    app.docs_writer = docs_system.docs_writer
    app.get_update_suggestions = docs_system.get_update_suggestions
    app.auto_update_docs = docs_system.auto_update_docs
    app.initial_docs_parse = docs_system.initial_docs_parse
    app.source_code_lookup = docs_system.source_code_lookup

    return docs_system
notification
NotificationAction dataclass

Represents an action button in a notification

Source code in toolboxv2/utils/extras/notification.py
19
20
21
22
23
24
25
@dataclass
class NotificationAction:
    """Represents an action button in a notification"""
    id: str
    label: str
    callback: Optional[Callable[[], Any]] = None
    is_default: bool = False
NotificationDetails dataclass

Expandable details for notifications

Source code in toolboxv2/utils/extras/notification.py
28
29
30
31
32
33
@dataclass
class NotificationDetails:
    """Expandable details for notifications"""
    title: str
    content: str
    data: Optional[Dict] = None
NotificationPosition

Position options for notifications

Source code in toolboxv2/utils/extras/notification.py
44
45
46
47
48
49
50
51
52
53
54
class NotificationPosition(Enum):
    """Position options for notifications"""
    TOP_LEFT = "top_left"
    TOP_CENTER = "top_center"
    TOP_RIGHT = "top_right"
    CENTER_LEFT = "center_left"
    CENTER = "center"
    CENTER_RIGHT = "center_right"
    BOTTOM_LEFT = "bottom_left"
    BOTTOM_CENTER = "bottom_center"
    BOTTOM_RIGHT = "bottom_right"
NotificationSystem

Cross-platform notification system with OS integration and tkinter fallback

Source code in toolboxv2/utils/extras/notification.py
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
class NotificationSystem(metaclass=Singleton):
    """
    Cross-platform notification system with OS integration and tkinter fallback
    """

    def __init__(self):
        self.platform = sys.platform.lower()
        self.fallback_to_tkinter = True
        self.sound_enabled = False
        self.default_timeout = 1500 # Add default timeout in milliseconds
        self.max_timeout = 30000
        self.default_position = NotificationPosition.TOP_RIGHT
        self._test_os_notifications()

    def _test_os_notifications(self):
        """Test if OS notifications are available"""
        try:
            if self.platform.startswith('win'):
                # Test Windows toast notifications
                try:
                    import win10toast
                    # Test if we can create a ToastNotifier without errors
                    try:
                        toaster = win10toast.ToastNotifier()
                        # Test if classAtom exists (common error)
                        if not hasattr(toaster, 'classAtom'):
                            print("⚠️  win10toast library has compatibility issues. Will use alternative methods.")
                            # Don't set fallback_to_tkinter = True, we have alternatives
                        self.fallback_to_tkinter = False
                    except AttributeError:
                        print("⚠️  win10toast library has issues. Using alternative Windows notification methods.")
                        self.fallback_to_tkinter = True
                except ImportError:
                    print("⚠️  Windows toast notifications not available. Install win10toast: pip install win10toast")
                    print("    Alternative: Will try built-in Windows notification methods.")
                    self.fallback_to_tkinter = True  # We still have alternatives
            elif self.platform.startswith('darwin'):
                # Test macOS notifications
                try:
                    result = subprocess.run(['which', 'osascript'],
                                            capture_output=True, text=True)
                    if result.returncode != 0:
                        raise FileNotFoundError
                    self.fallback_to_tkinter = False
                except:
                    print("⚠️  macOS notifications not available. osascript not found.")
                    self.fallback_to_tkinter = True

            elif self.platform.startswith('linux'):
                # Test Linux notifications
                try:
                    result = subprocess.run(['which', 'notify-send'],
                                            capture_output=True, text=True)
                    if result.returncode != 0:
                        raise FileNotFoundError
                    self.fallback_to_tkinter = False
                except:
                    print(
                        "⚠️  Linux notifications not available. Install libnotify-bin: sudo apt install libnotify-bin")
                    self.fallback_to_tkinter = True
            else:
                print("⚠️  Unknown platform. Using tkinter fallback.")
                self.fallback_to_tkinter = True

        except Exception as e:
            print(f"⚠️  OS notification test failed: {e}. Using tkinter fallback.")
            self.fallback_to_tkinter = True

    def show_notification(self,
                          title: str,
                          message: str,
                          notification_type: NotificationType = NotificationType.INFO,
                          actions: List[NotificationAction] = None,
                          details: NotificationDetails = None,
                          timeout: int = None,
                          play_sound: bool = False,
                          position: NotificationPosition = None) -> Optional[str]:
        """
        Show a notification with optional actions and details

        Args:
            title (str): Title of the notification
            message (str): Main message of the notification
            notification_type (NotificationType): Type of notification
            actions (List[NotificationAction]): List of action buttons
            details (NotificationDetails): Expandable details
            timeout (int): Timeout in milliseconds
            play_sound (bool): Whether to play a sound
            position (NotificationPosition): Position on screen

        Returns the ID of the selected action, or None if dismissed
        """
        # Handle position configuration
        if position is None:
            position = self.default_position

        if timeout is None:
            timeout = self.default_timeout
        elif timeout > self.max_timeout:
            timeout = self.max_timeout
        elif timeout < 0:
            timeout = 0

        if play_sound and self.sound_enabled:
            self._play_notification_sound(notification_type)

        if self.fallback_to_tkinter or actions or details:
            # Use tkinter for complex notifications or as fallback
            return self._show_tkinter_notification(title, message, notification_type,
                                                   actions, details, timeout, position)
        else:
            # Use OS notification for simple notifications
            return self._show_os_notification(title, message, notification_type, timeout)

    def set_default_timeout(self, timeout_ms: int):
        """Set default timeout for notifications"""
        if timeout_ms < 0:
            self.default_timeout = 0  # No timeout
        elif timeout_ms > self.max_timeout:
            self.default_timeout = self.max_timeout
        else:
            self.default_timeout = timeout_ms

    def set_max_timeout(self, max_timeout_ms: int):
        """Set maximum allowed timeout"""
        if max_timeout_ms > 0:
            self.max_timeout = max_timeout_ms

    def set_default_position(self, position: NotificationPosition):
        """Set default position for notifications"""
        self.default_position = position

    def _show_os_notification(self, title: str, message: str,
                              notification_type: NotificationType, timeout: int) -> None:
        """Show OS native notification"""

        try:
            if self.platform.startswith('win'):
                self._show_windows_notification(title, message, notification_type, timeout)
            elif self.platform.startswith('darwin'):
                self._show_macos_notification(title, message, notification_type, timeout)
            elif self.platform.startswith('linux'):
                self._show_linux_notification(title, message, notification_type, timeout)
        except Exception as e:
            print(f"⚠️  OS notification failed: {e}. Falling back to tkinter.")
            return self._show_tkinter_notification(title, message, notification_type)

    def _show_windows_notification(self, title: str, message: str,
                                               notification_type: NotificationType, timeout):
        """Alternative Windows notification using ctypes"""
        try:
            import ctypes
            from ctypes import wintypes

            # Try using Windows 10+ notification API via PowerShell
            try:
                icon_map = {
                    NotificationType.INFO: "Information",
                    NotificationType.SUCCESS: "success",
                    NotificationType.WARNING: "Warning",
                    NotificationType.ERROR: "Error",
                    NotificationType.QUESTION: "Question"
                }

                icon_type = icon_map.get(notification_type, "Information")

                # PowerShell script to show notification
                ps_script = f'''
                [Windows.UI.Notifications.ToastNotificationManager, Windows.UI.Notifications, ContentType = WindowsRuntime] | Out-Null
                [Windows.UI.Notifications.ToastNotification, Windows.UI.Notifications, ContentType = WindowsRuntime] | Out-Null
                [Windows.Data.Xml.Dom.XmlDocument, Windows.Data.Xml.Dom.XmlDocument, ContentType = WindowsRuntime] | Out-Null

                $template = @"
                <toast>
                    <visual>
                        <binding template="ToastGeneric">
                            <text>{title}</text>
                            <text>{message}</text>
                        </binding>
                    </visual>
                </toast>
                "@

                $xml = New-Object Windows.Data.Xml.Dom.XmlDocument
                $xml.LoadXml($template)
                $toast = New-Object Windows.UI.Notifications.ToastNotification $xml
                [Windows.UI.Notifications.ToastNotificationManager]::CreateToastNotifier("Python App").Show($toast)
                '''

                result = subprocess.run(['powershell', '-Command', text_save(ps_script)],
                                        capture_output=True, text=True, timeout=5)

                if result.returncode == 0:
                    return

            except Exception:
                pass

            # Fallback to simple MessageBox
            MB_ICONINFORMATION = 0x40
            MB_ICONWARNING = 0x30
            MB_ICONERROR = 0x10
            MB_ICONQUESTION = 0x20

            icon_map = {
                NotificationType.INFO: MB_ICONINFORMATION,
                NotificationType.SUCCESS: MB_ICONINFORMATION,
                NotificationType.WARNING: MB_ICONWARNING,
                NotificationType.ERROR: MB_ICONERROR,
                NotificationType.QUESTION: MB_ICONQUESTION
            }

            icon = icon_map.get(notification_type, MB_ICONINFORMATION)

            # Show MessageBox in separate thread to avoid blocking
            def show_messagebox():
                try:
                    ctypes.windll.user32.MessageBoxW(0, message, title, icon)
                except:
                    pass

            threading.Thread(target=show_messagebox, daemon=True).start()

        except Exception:
            raise Exception("All Windows notification methods failed")

    def _show_macos_notification(self, title: str, message: str,
                                 notification_type: NotificationType, timeout: int):
        """macOS notification using osascript"""
        try:
            script = f'''
                display notification "{message}" with title "{title}"
            '''
            subprocess.run(['osascript', '-e', text_save(script)], check=True)
        except Exception as e:
            raise Exception(f"macOS notification failed: {e}")

    def _show_linux_notification(self, title: str, message: str,
                                 notification_type: NotificationType, timeout: int):
        """Linux notification using notify-send"""
        try:
            urgency = "normal"
            if notification_type == NotificationType.ERROR:
                urgency = "critical"
            elif notification_type == NotificationType.WARNING:
                urgency = "normal"

            icon = self._get_linux_icon(notification_type)

            subprocess.run([
                'notify-send',
                f'--urgency={urgency}',
                f'--expire-time={timeout}',
                f'--icon={icon}',
                text_save(title),
                text_save(message)
            ], check=True)
        except Exception as e:
            raise Exception(f"Linux notification failed: {e}")

    def _show_tkinter_notification(self, title: str, message: str,
                                   notification_type: NotificationType,
                                   actions: List[NotificationAction] = None,
                                   details: NotificationDetails = None,
                                   timeout: int = 5000,
                                   position: NotificationPosition = NotificationPosition.CENTER) -> Optional[str]:
        """Modern dark-themed tkinter notification dialog"""

        # Use a queue to communicate between threads
        result_queue = queue.Queue()

        def run_notification():
            try:
                import tkinter as tk
                from tkinter import ttk

                # Create root window
                root = tk.Tk()
                root.withdraw()  # Hide the root window

                # Create notification window
                window = tk.Toplevel(root)

                # Dark theme colors
                bg_color = "#2b2b2b"
                fg_color = "#ffffff"
                accent_color = self._get_accent_color(notification_type)
                button_color = "#404040"
                button_hover = "#505050"
                border_color = "#404040"

                # Remove window decorations for custom styling
                window.overrideredirect(True)
                window.configure(bg=border_color)

                # Variables for dragging (use instance variables to avoid threading issues)
                window.drag_data = {"x": 0, "y": 0}
                window.details_visible = False
                window.result = None

                # Create main container with border
                border_frame = tk.Frame(window, bg=border_color, padx=1, pady=1)
                border_frame.pack(fill=tk.BOTH, expand=True)

                main_container = tk.Frame(border_frame, bg=bg_color)
                main_container.pack(fill=tk.BOTH, expand=True)

                # Title bar for dragging and close button
                title_bar = tk.Frame(main_container, bg=accent_color, height=25)
                title_bar.pack(fill=tk.X, side=tk.TOP)
                title_bar.pack_propagate(False)

                # Window title in title bar
                title_label = tk.Label(title_bar, text="Notification",
                                       font=("Arial", 9), bg=accent_color, fg=fg_color)
                title_label.pack(side=tk.LEFT, padx=8, pady=4)

                # Close button
                def close_window():
                    window.result = None
                    result_queue.put(window.result)
                    root.quit()
                    root.destroy()

                close_btn = tk.Label(title_bar, text="✕", font=("Arial", 10, "bold"),
                                     bg=accent_color, fg=fg_color, cursor="hand2",
                                     padx=8, pady=2)
                close_btn.pack(side=tk.RIGHT)
                close_btn.bind("<Button-1>", lambda e: close_window())
                close_btn.bind("<Enter>", lambda e: close_btn.config(bg=self._lighten_color(accent_color, -0.2)))
                close_btn.bind("<Leave>", lambda e: close_btn.config(bg=accent_color))

                # Make title bar draggable
                def start_drag(event):
                    window.drag_data["x"] = event.x
                    window.drag_data["y"] = event.y

                def on_drag(event):
                    x = window.winfo_x() + (event.x - window.drag_data["x"])
                    y = window.winfo_y() + (event.y - window.drag_data["y"])
                    window.geometry(f"+{x}+{y}")

                title_bar.bind("<Button-1>", start_drag)
                title_bar.bind("<B1-Motion>", on_drag)
                title_label.bind("<Button-1>", start_drag)
                title_label.bind("<B1-Motion>", on_drag)

                # Content frame
                content_frame = tk.Frame(main_container, bg=bg_color, padx=15, pady=12)
                content_frame.pack(fill=tk.BOTH, expand=True)

                # Header with icon and title (more compact)
                header_frame = tk.Frame(content_frame, bg=bg_color)
                header_frame.pack(fill=tk.X, pady=(0, 8))

                # Notification type icon (smaller)
                icon_label = tk.Label(header_frame, text=self._get_emoji_icon(notification_type),
                                      font=("Arial", 16), bg=bg_color, fg=accent_color)
                icon_label.pack(side=tk.LEFT, padx=(0, 8))

                # Title (smaller font)
                title_text = tk.Label(header_frame, text=title, font=("Arial", 11, "bold"),
                                      bg=bg_color, fg=fg_color, wraplength=280)
                title_text.pack(side=tk.LEFT, fill=tk.X, expand=True, anchor="w")

                # Message (more compact)
                message_label = tk.Label(content_frame, text=message, font=("Arial", 9),
                                         bg=bg_color, fg=fg_color, wraplength=320, justify=tk.LEFT)
                message_label.pack(fill=tk.X, pady=(0, 8))

                # Details section (expandable) - initially hidden
                details_frame = None
                details_text_widget = None

                if details:
                    details_container = tk.Frame(content_frame, bg=bg_color)
                    details_container.pack(fill=tk.X, pady=(0, 8))

                    def toggle_details():
                        nonlocal details_frame, details_text_widget

                        if not window.details_visible:
                            # Show details
                            if details_frame is None:
                                details_frame = tk.Frame(details_container, bg=bg_color)
                                details_frame.pack(fill=tk.X, pady=(4, 0))

                                # Create scrollable text area
                                text_frame = tk.Frame(details_frame, bg="#1e1e1e")
                                text_frame.pack(fill=tk.X, pady=(0, 0))

                                details_text_widget = tk.Text(text_frame, height=5, bg="#1e1e1e", fg=fg_color,
                                                              border=0, wrap=tk.WORD, font=("Consolas", 8),
                                                              padx=8, pady=6)
                                details_text_widget.pack(fill=tk.X)

                                detail_content = f"{details.title}\n{'-' * min(40, len(details.title))}\n{details.content}"
                                if details.data:
                                    detail_content += f"\n\nData:\n{json.dumps(details.data, indent=2)}"

                                details_text_widget.insert(tk.END, detail_content)
                                details_text_widget.config(state=tk.DISABLED)

                            details_btn.config(text="▼ Hide Details")
                            details_frame.pack(fill=tk.X, pady=(4, 0))
                            window.details_visible = True

                            # Resize window
                            window.update_idletasks()
                            new_height = window.winfo_reqheight()
                            window.geometry(f"380x{new_height}")
                        else:
                            # Hide details
                            details_btn.config(text="▶ Show Details")
                            if details_frame:
                                details_frame.pack_forget()
                            window.details_visible = False

                            # Resize window back
                            window.update_idletasks()
                            new_height = window.winfo_reqheight()
                            window.geometry(f"380x{new_height}")

                    details_btn = tk.Button(details_container, text="▶ Show Details",
                                            command=toggle_details, bg=button_color, fg=fg_color,
                                            border=0, font=("Arial", 8), relief=tk.FLAT, cursor="hand2")
                    details_btn.pack(anchor=tk.W)

                # Action buttons (more compact)
                if actions:
                    button_frame = tk.Frame(content_frame, bg=bg_color)
                    button_frame.pack(fill=tk.X, side=tk.BOTTOM, pady=(8, 0))

                    for i, action in enumerate(actions):
                        def make_callback(action_id, callback):
                            def callback_wrapper():
                                window.result = action_id
                                result_queue.put(window.result)
                                if callback:
                                    # Run callback in separate thread
                                    threading.Thread(target=callback, daemon=True).start()
                                root.quit()
                                root.destroy()

                            return callback_wrapper

                        btn_bg = accent_color if action.is_default else button_color
                        btn = tk.Button(button_frame, text=action.label,
                                        command=make_callback(action.id, action.callback),
                                        bg=btn_bg, fg=fg_color, border=0,
                                        font=("Arial", 9), relief=tk.FLAT,
                                        padx=12, pady=6, cursor="hand2")
                        btn.pack(side=tk.RIGHT, padx=(4, 0))

                        # Hover effects
                        def on_enter(e, btn=btn, color=btn_bg):
                            btn.config(bg=self._lighten_color(color))

                        def on_leave(e, btn=btn, color=btn_bg):
                            btn.config(bg=color)

                        btn.bind("<Enter>", on_enter)
                        btn.bind("<Leave>", on_leave)
                else:
                    # Default OK button
                    button_frame = tk.Frame(content_frame, bg=bg_color)
                    button_frame.pack(fill=tk.X, side=tk.BOTTOM, pady=(8, 0))

                    def ok_clicked():
                        window.result = "ok"
                        result_queue.put(window.result)
                        root.quit()
                        root.destroy()

                    ok_btn = tk.Button(button_frame, text="OK", command=ok_clicked,
                                       bg=accent_color, fg=fg_color, border=0,
                                       font=("Arial", 9), relief=tk.FLAT,
                                       padx=12, pady=6, cursor="hand2")
                    ok_btn.pack(side=tk.RIGHT)

                # Set initial window size (slimmer)
                base_height = 150
                if timeout > 5000:
                    base_height += 20  # Add space for timeout indicator

                # Center window on screen
                # Position window based on specified position
                window.update()
                screen_width = window.winfo_screenwidth()
                screen_height = window.winfo_screenheight()
                window_width = 380
                window_height = window.winfo_height()

                # Calculate position based on enum
                margin = 20  # Margin from screen edges

                if position == NotificationPosition.TOP_LEFT:
                    x, y = margin, margin
                elif position == NotificationPosition.TOP_CENTER:
                    x, y = (screen_width - window_width) // 2, margin
                elif position == NotificationPosition.TOP_RIGHT:
                    x, y = screen_width - window_width - margin, margin
                elif position == NotificationPosition.CENTER_LEFT:
                    x, y = margin, (screen_height - window_height) // 2
                elif position == NotificationPosition.CENTER:
                    x, y = (screen_width - window_width) // 2, (screen_height - window_height) // 2 - 50
                elif position == NotificationPosition.CENTER_RIGHT:
                    x, y = screen_width - window_width - margin, (screen_height - window_height) // 2
                elif position == NotificationPosition.BOTTOM_LEFT:
                    x, y = margin, screen_height - window_height - margin - 50  # Account for taskbar
                elif position == NotificationPosition.BOTTOM_CENTER:
                    x, y = (screen_width - window_width) // 2, screen_height - window_height - margin - 50
                elif position == NotificationPosition.BOTTOM_RIGHT:
                    x, y = screen_width - window_width - margin, screen_height - window_height - margin - 50
                else:
                    # Default to center
                    x, y = (screen_width - window_width) // 2, (screen_height - window_height) // 2 - 50
                window.geometry(f"{window_width}x{window_height}+{x}+{y}")
                window.update()
                # Always on top and focus
                window.attributes('-topmost', True)
                window.focus_force()

                # Auto-close after timeout (if no actions)
                if not actions:
                    # Auto-close after timeout (for all notifications if timeout > 0)
                    if timeout > 0:
                        def create_auto_close():
                            def auto_close_handler():
                                try:
                                    if root.winfo_exists():
                                        window.result = 'timeout'
                                        result_queue.put('timeout')
                                        root.quit()
                                        root.destroy()
                                except tk.TclError:
                                    pass  # Window already destroyed
                                except Exception:
                                    pass  # Handle any other errors silently

                            root.after(timeout, auto_close_handler)

                        create_auto_close()

                    # Add timeout indicator if timeout > 10 seconds
                    # Alternative: Progress bar timeout indicator (replace the text version above)
                    if timeout > 5000:
                        timeout_frame = tk.Frame(content_frame, bg=bg_color)
                        timeout_frame.pack(fill=tk.X, pady=(2, 4))

                        # Progress bar for visual timeout
                        progress_bg = tk.Frame(timeout_frame, bg="#444444", height=4)
                        progress_bg.pack(fill=tk.X, pady=(0, 2))

                        progress_bar = tk.Frame(progress_bg, bg="#666666", height=4)
                        progress_bar.place(x=0, y=0, relwidth=1.0, height=4)

                        # Timeout text
                        timeout_label = tk.Label(timeout_frame,
                                                 text=f"⏱️ Auto-closes in {timeout // 1000}s",
                                                 font=("Arial", 8), bg=bg_color, fg="#888888")
                        timeout_label.pack(anchor=tk.E)

                        def setup_progress_countdown():
                            total_time = timeout // 1000
                            remaining = [total_time]

                            def update_progress():
                                try:
                                    if remaining[0] > 0 and root and root.winfo_exists():
                                        # Update text
                                        timeout_label.config(text=f"⏱️ Auto-closes in {remaining[0]}s")

                                        # Update progress bar
                                        progress_width = remaining[0] / total_time
                                        progress_bar.place(relwidth=progress_width)

                                        remaining[0] -= 1
                                        root.after(1000, update_progress)
                                    elif root and root.winfo_exists():
                                        timeout_label.config(text="⏱️ Closing...")
                                        progress_bar.place(relwidth=0)
                                except (tk.TclError, AttributeError):
                                    pass

                            root.after(1000, update_progress)

                        setup_progress_countdown()

                # Handle escape key
                def on_escape(event):
                    close_window()

                window.bind('<Escape>', on_escape)
                window.focus_set()

                # Start the GUI main loop
                root.mainloop()

            except Exception as e:
                print(f"⚠️  Tkinter notification error: {e}")
                result_queue.put(None)

        # Run notification in the main thread if possible, otherwise in a separate thread
        if threading.current_thread() is threading.main_thread():
            run_notification()
        else:
            # If not in main thread, we need to handle this differently
            gui_thread = threading.Thread(target=run_notification, daemon=True)
            gui_thread.start()
            gui_thread.join(timeout=30)  # Don't wait forever

        # Get result from queue
        try:
            if actions:
                return result_queue.get(timeout=1)
            return None
        except queue.Empty:
            return None

    def _play_notification_sound(self, notification_type: NotificationType):
        """Play appropriate sound for notification type"""
        try:
            if notification_type == NotificationType.ERROR:
                self._play_sound(frequency=800, duration=0.5)
            elif notification_type == NotificationType.WARNING:
                self._play_sound(frequency=600, duration=0.3)
            elif notification_type == NotificationType.SUCCESS:
                self._play_sound(frequency=1000, duration=0.2)
            else:
                self._play_sound(frequency=700, duration=0.3)
        except:
            pass  # Don't let sound errors break notifications

    def _play_sound(self, frequency: int = 800, duration: float = 0.3):
        """Play notification sound"""

        def play():
            try:
                if self.platform.startswith('win'):
                    import winsound
                    winsound.Beep(frequency, int(duration * 1000))
                elif self.platform.startswith('darwin'):
                    subprocess.run(['afplay', '/System/Library/Sounds/Ping.aiff'],
                                   check=True, capture_output=True)
                elif self.platform.startswith('linux'):
                    try:
                        subprocess.run(['paplay', '--raw', '--format=s16le',
                                        '--rate=44100', '--channels=1'],
                                       input=self._generate_tone_data(frequency, duration, 44100),
                                       check=True, timeout=2)
                    except:
                        print('\a')  # Fallback to system bell
                else:
                    print('\a')
            except:
                print('\a')  # Ultimate fallback

        # Play sound in separate thread to not block UI
        threading.Thread(target=play, daemon=True).start()

    def _generate_tone_data(self, frequency: int, duration: float, sample_rate: int = 44100) -> bytes:
        """Generate raw audio data for a sine wave tone"""
        import math
        import struct

        num_samples = int(sample_rate * duration)
        tone_data = []

        for i in range(num_samples):
            t = i / sample_rate
            fade = min(1.0, t * 10, (duration - t) * 10)
            sample = int(16384 * fade * math.sin(2 * math.pi * frequency * t))
            tone_data.append(struct.pack('<h', sample))

        return b''.join(tone_data)

    def _get_emoji_icon(self, notification_type: NotificationType) -> str:
        """Get emoji icon for notification type"""
        icons = {
            NotificationType.INFO: "ℹ️",
            NotificationType.SUCCESS: "✅",
            NotificationType.WARNING: "⚠️",
            NotificationType.ERROR: "❌",
            NotificationType.QUESTION: "❓"
        }
        return icons.get(notification_type, "📢")

    def _get_accent_color(self, notification_type: NotificationType) -> str:
        """Get accent color for notification type"""
        colors = {
            NotificationType.INFO: "#3498db",
            NotificationType.SUCCESS: "#27ae60",
            NotificationType.WARNING: "#f39c12",
            NotificationType.ERROR: "#e74c3c",
            NotificationType.QUESTION: "#9b59b6"
        }
        return colors.get(notification_type, "#3498db")

    def _lighten_color(self, color: str, factor: float = 0.2) -> str:
        """Lighten or darken a hex color"""
        try:
            color = color.lstrip('#')
            rgb = tuple(int(color[i:i + 2], 16) for i in (0, 2, 4))
            if factor > 0:
                # Lighten
                rgb = tuple(min(255, int(c + (255 - c) * factor)) for c in rgb)
            else:
                # Darken
                rgb = tuple(max(0, int(c * (1 + factor))) for c in rgb)
            return f"#{rgb[0]:02x}{rgb[1]:02x}{rgb[2]:02x}"
        except:
            return color

    def _get_icon_path(self, notification_type: NotificationType) -> Optional[str]:
        """Get icon path for Windows notifications"""
        return None

    def _get_linux_icon(self, notification_type: NotificationType) -> str:
        """Get Linux system icon name"""
        icons = {
            NotificationType.INFO: "dialog-information",
            NotificationType.SUCCESS: "dialog-information",
            NotificationType.WARNING: "dialog-warning",
            NotificationType.ERROR: "dialog-error",
            NotificationType.QUESTION: "dialog-question"
        }
        return icons.get(notification_type, "dialog-information")
set_default_position(position)

Set default position for notifications

Source code in toolboxv2/utils/extras/notification.py
185
186
187
def set_default_position(self, position: NotificationPosition):
    """Set default position for notifications"""
    self.default_position = position
set_default_timeout(timeout_ms)

Set default timeout for notifications

Source code in toolboxv2/utils/extras/notification.py
171
172
173
174
175
176
177
178
def set_default_timeout(self, timeout_ms: int):
    """Set default timeout for notifications"""
    if timeout_ms < 0:
        self.default_timeout = 0  # No timeout
    elif timeout_ms > self.max_timeout:
        self.default_timeout = self.max_timeout
    else:
        self.default_timeout = timeout_ms
set_max_timeout(max_timeout_ms)

Set maximum allowed timeout

Source code in toolboxv2/utils/extras/notification.py
180
181
182
183
def set_max_timeout(self, max_timeout_ms: int):
    """Set maximum allowed timeout"""
    if max_timeout_ms > 0:
        self.max_timeout = max_timeout_ms
show_notification(title, message, notification_type=NotificationType.INFO, actions=None, details=None, timeout=None, play_sound=False, position=None)

Show a notification with optional actions and details

Parameters:

Name Type Description Default
title str

Title of the notification

required
message str

Main message of the notification

required
notification_type NotificationType

Type of notification

INFO
actions List[NotificationAction]

List of action buttons

None
details NotificationDetails

Expandable details

None
timeout int

Timeout in milliseconds

None
play_sound bool

Whether to play a sound

False
position NotificationPosition

Position on screen

None

Returns the ID of the selected action, or None if dismissed

Source code in toolboxv2/utils/extras/notification.py
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
def show_notification(self,
                      title: str,
                      message: str,
                      notification_type: NotificationType = NotificationType.INFO,
                      actions: List[NotificationAction] = None,
                      details: NotificationDetails = None,
                      timeout: int = None,
                      play_sound: bool = False,
                      position: NotificationPosition = None) -> Optional[str]:
    """
    Show a notification with optional actions and details

    Args:
        title (str): Title of the notification
        message (str): Main message of the notification
        notification_type (NotificationType): Type of notification
        actions (List[NotificationAction]): List of action buttons
        details (NotificationDetails): Expandable details
        timeout (int): Timeout in milliseconds
        play_sound (bool): Whether to play a sound
        position (NotificationPosition): Position on screen

    Returns the ID of the selected action, or None if dismissed
    """
    # Handle position configuration
    if position is None:
        position = self.default_position

    if timeout is None:
        timeout = self.default_timeout
    elif timeout > self.max_timeout:
        timeout = self.max_timeout
    elif timeout < 0:
        timeout = 0

    if play_sound and self.sound_enabled:
        self._play_notification_sound(notification_type)

    if self.fallback_to_tkinter or actions or details:
        # Use tkinter for complex notifications or as fallback
        return self._show_tkinter_notification(title, message, notification_type,
                                               actions, details, timeout, position)
    else:
        # Use OS notification for simple notifications
        return self._show_os_notification(title, message, notification_type, timeout)
NotificationType

Types of notifications

Source code in toolboxv2/utils/extras/notification.py
36
37
38
39
40
41
42
class NotificationType(Enum):
    """Types of notifications"""
    INFO = "info"
    SUCCESS = "success"
    WARNING = "warning"
    ERROR = "error"
    QUESTION = "question"
ask_question(title, message, yes_callback=None, no_callback=None, **kwargs)

Ask a yes/no question

Source code in toolboxv2/utils/extras/notification.py
817
818
819
820
821
822
823
824
825
826
827
828
829
830
def ask_question(title: str, message: str,
                 yes_callback: Callable = None,
                 no_callback: Callable = None, **kwargs) -> Optional[str]:
    """Ask a yes/no question"""
    notifier = create_notification_system()

    actions = [
        NotificationAction("yes", "Yes", yes_callback, is_default=True),
        NotificationAction("no", "No", no_callback)
    ]

    return notifier.show_notification(
        title, message, NotificationType.QUESTION, actions=actions, **kwargs
    )
create_notification_system()

Create and return a notification system instance

Source code in toolboxv2/utils/extras/notification.py
788
789
790
def create_notification_system() -> NotificationSystem:
    """Create and return a notification system instance"""
    return NotificationSystem()
example_notifications()

Example notification scenarios with better timing

Source code in toolboxv2/utils/extras/notification.py
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
def example_notifications():
    """Example notification scenarios with better timing"""

    notifier = create_notification_system()

    # Simple notification
    print("1. Simple info notification...")
    notifier.show_notification(
        title="Welcome!",
        message="Application started successfully.",
        notification_type=NotificationType.INFO
    )

    time.sleep(2)

    # Success notification
    print("2. Success notification...")
    notifier.show_notification(
        title="Task Complete",
        message="Your file has been processed successfully.",
        notification_type=NotificationType.SUCCESS
    )

    time.sleep(2)

    # Warning with details
    print("3. Warning with expandable details...")
    details = NotificationDetails(
        title="Performance Warning",
        content="The system is running low on memory. Consider closing some applications to free up resources.",
        data={
            "memory_usage": "85%",
            "available_memory": "2.1 GB",
            "total_memory": "16 GB",
            "top_processes": ["Chrome", "Visual Studio", "Photoshop"]
        }
    )

    notifier.show_notification(
        title="System Warning",
        message="High memory usage detected.",
        notification_type=NotificationType.WARNING,
        details=details
    )

    time.sleep(2)

    # Interactive notification with actions
    print("4. Interactive notification with actions...")

    def handle_update():
        print("🔄 Update initiated!")
        time.sleep(1)
        notifier.show_notification(
            title="Update Complete",
            message="Application has been updated to version 2.1.0.",
            notification_type=NotificationType.SUCCESS
        )

    def handle_remind_later():
        print("⏰ Reminder set for later!")
        notifier.show_notification(
            title="Reminder Set",
            message="You'll be reminded about the update in 1 hour.",
            notification_type=NotificationType.INFO
        )

    actions = [
        NotificationAction("update", "Update Now", handle_update, is_default=True),
        NotificationAction("later", "Remind Later", handle_remind_later),
        NotificationAction("skip", "Skip Version", lambda: print("❌ Update skipped"))
    ]

    selected_action = notifier.show_notification(
        title="Update Available",
        message="Version 2.1.0 is ready to install with bug fixes and new features.",
        notification_type=NotificationType.QUESTION,
        actions=actions,
        details=NotificationDetails(
            title="Update Information",
            content="This update includes security patches, performance improvements, and new features.",
            data={
                "version": "2.1.0",
                "size": "25.3 MB",
                "release_date": "2024-01-15",
                "changelog": [
                    "Fixed memory leak in file processing",
                    "Added dark mode support",
                    "Improved startup time by 40%",
                    "Updated dependencies for security"
                ]
            }
        )
    )

    print(f"✅ Selected action: {selected_action}")

    print("5. Testing different positions...")

    positions_to_test = [
        (NotificationPosition.TOP_RIGHT, "Top Right"),
        (NotificationPosition.BOTTOM_LEFT, "Bottom Left"),
        (NotificationPosition.TOP_CENTER, "Top Center"),
        (NotificationPosition.CENTER_RIGHT, "Center Right")
    ]
    notifier.fallback_to_tkinter = True
    for position, pos_name in positions_to_test:
        notifier.show_notification(
            title=f"{pos_name} Notification",
            message=f"This notification appears at {pos_name.lower()}",
            notification_type=NotificationType.INFO,
            position=position,
            timeout=2000
        )
        time.sleep(0.5)
quick_error(title, message, **kwargs)

Quick error notification

Source code in toolboxv2/utils/extras/notification.py
811
812
813
814
def quick_error(title: str, message: str, **kwargs):
    """Quick error notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.ERROR, **kwargs)
quick_info(title, message, **kwargs)

Quick info notification

Source code in toolboxv2/utils/extras/notification.py
793
794
795
796
def quick_info(title: str, message: str, **kwargs):
    """Quick info notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.INFO, **kwargs)
quick_success(title, message, **kwargs)

Quick success notification

Source code in toolboxv2/utils/extras/notification.py
799
800
801
802
def quick_success(title: str, message: str, **kwargs):
    """Quick success notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.SUCCESS, **kwargs)
quick_warning(title, message, **kwargs)

Quick warning notification

Source code in toolboxv2/utils/extras/notification.py
805
806
807
808
def quick_warning(title: str, message: str, **kwargs):
    """Quick warning notification"""
    notifier = create_notification_system()
    notifier.show_notification(title, message, NotificationType.WARNING, **kwargs)
reqbuilder
generate_requirements(folder, output_file)

Generates requirements.txt for the specified folder using pipreqs.

Source code in toolboxv2/utils/extras/reqbuilder.py
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
def generate_requirements(folder: str, output_file: str):
    """Generates requirements.txt for the specified folder using pipreqs."""
    print(folder, output_file, os.path.abspath(os.curdir))
    print("Not Implemented ")
    """try:
        from pipreqs.pipreqs import get_all_imports
    except ImportError:
        subprocess.run([sys.executable, "-m", "pip", "install", "pipreqs"], check=True)
    from pipreqs.pipreqs import get_all_imports
    imports = set(get_all_imports(os.path.abspath(folder)))
    imports.remove('toolboxv2') if 'toolboxv2' in imports else None
    with open(os.path.abspath(output_file), "w") as f:
        f.write("\n".join(imports))"""
run_pipeline(base_dir)

Runs the entire pipeline to generate requirements files.

Source code in toolboxv2/utils/extras/reqbuilder.py
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
def run_pipeline(base_dir: str):
    """Runs the entire pipeline to generate requirements files."""
    toolbox_path = os.path.join(base_dir, "toolboxv2")
    utils_path = os.path.join(toolbox_path, "utils")
    mini_req_file = os.path.join(base_dir, "requirements_mini.txt")
    extras_req_file = os.path.join(base_dir, "requirements_tests.txt")

    # Step 1: Generate minimal requirements
    print("Step 1/2: ")
    generate_requirements(utils_path, mini_req_file)

    # Step 2: Generate extended requirements
    print("Step 2/2: ")
    extras_path = os.path.join(toolbox_path, "tests")
    generate_requirements(extras_path, extras_req_file)

install_support

Complete TB Language Setup - Build executable - Setup file associations - Install VS Code extension - Install PyCharm plugin

TBSetup

Complete TB Language setup manager

Source code in toolboxv2/utils/tbx/install_support.py
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
class TBSetup:
    """Complete TB Language setup manager"""

    def __init__(self):
        self.root = Path(__file__).parent
        self.system = platform.system()

    def setup_all(self):
        """Run complete setup"""
        print("═" * 70)
        print("  TB Language - Complete Setup")
        print("═" * 70)
        print()

        success = True

        # Step 1: Build
        if not self.build_executable():
            print("❌ Build failed!")
            return False

        # Step 2: System integration
        if not self.setup_system_integration():
            print("⚠️  System integration failed (optional)")
            success = False

        # Step 3: VS Code extension
        if not self.setup_vscode():
            print("⚠️  VS Code extension setup failed (optional)")
            success = False

        # Step 4: PyCharm plugin
        if not self.setup_pycharm():
            print("⚠️  PyCharm plugin setup failed (optional)")
            success = False

        print()
        print("═" * 70)
        if success:
            print("  ✓ Setup Complete!")
        else:
            print("  ⚠️  Setup completed with warnings")
        print("═" * 70)
        print()
        print("Next steps:")
        print("  1. Restart PyCharm and VS Code (if open)")
        print("  2. Create a test file: test.tbx")
        print("  3. Run it: tb run test.tbx")
        print("  4. Or double-click test.tbx to run")
        print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
        print()

        return success

    def build_executable(self):
        """Step 1: Build TB Language"""
        print("Step 1/4: Building TB Language...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
            "build"
        ])

        if result.returncode != 0:
            return False

        print("✓ Build successful")
        print()
        return True

    def setup_system_integration(self):
        """Step 2: System integration"""
        print("Step 2/4: Setting up system integration...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_setup.py"),
            "install"
        ])

        print()
        return result.returncode == 0

    def setup_vscode(self):
        """Step 3: VS Code extension"""
        print("Step 3/4: Installing VS Code extension...")
        print("-" * 70)

        vscode_ext = self.root / "tb-lang-vscode"
        if not vscode_ext.exists():
            print("⚠️  VS Code extension directory not found")
            print()
            return False

        try:
            # Check if npm is available
            subprocess.run(["npm", "--version"],
                           capture_output=True, check=True)

            # Install dependencies
            print("  Installing npm dependencies...")
            subprocess.run(["npm", "install"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Compile TypeScript
            print("  Compiling TypeScript...")
            subprocess.run(["npm", "run", "compile"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Try to install to VS Code
            print("  Installing to VS Code...")
            result = subprocess.run([
                "code", "--install-extension", str(vscode_ext.resolve())
            ], capture_output=True)

            if result.returncode == 0:
                print("✓ VS Code extension installed")
                print()
                return True
            else:
                print("⚠️  Could not auto-install to VS Code")
                print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
                print()
                return False

        except FileNotFoundError as e:
            print(f"⚠️  Tool not found: {e}")
            print("   npm: https://nodejs.org/")
            print("   VS Code: https://code.visualstudio.com/")
            print()
            return False
        except subprocess.CalledProcessError as e:
            print(f"⚠️  Command failed: {e}")
            print()
            return False

    def setup_pycharm(self):
        """Step 4: PyCharm plugin"""
        print("Step 4/4: Installing PyCharm plugin...")
        print("-" * 70)

        pycharm_plugin = self.root / "tb-lang-pycharm"
        if not pycharm_plugin.exists():
            print("⚠️  PyCharm plugin directory not found")
            print("   Creating plugin structure...")
            if not self.create_pycharm_plugin():
                print()
                return False

        try:
            # Build plugin JAR
            print("  Building PyCharm plugin...")
            if not self.build_pycharm_plugin():
                print("⚠️  Plugin build failed")
                print()
                return False

            # Install to PyCharm
            print("  Installing to PyCharm...")
            if not self.install_pycharm_plugin():
                print("⚠️  Auto-install failed")
                print()
                return False

            print("✓ PyCharm plugin installed")
            print("  Please restart PyCharm to activate the plugin")
            print()
            return True

        except Exception as e:
            print(f"⚠️  Error: {e}")
            print()
            return False

    def create_pycharm_plugin(self):
        """Create PyCharm plugin structure"""
        plugin_dir = self.root / "tb-lang-pycharm"
        plugin_dir.mkdir(exist_ok=True)

        # Create directory structure
        (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
        (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

        return True

    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False

    def install_pycharm_plugin(self):
        """Install plugin to PyCharm"""
        plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

        if not plugin_jar.exists():
            print("  Plugin JAR not found")
            return False

        # Find PyCharm config directory
        pycharm_dirs = self.find_pycharm_config_dirs()

        if not pycharm_dirs:
            print("  PyCharm installation not found")
            print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
            return False

        # Install to all found PyCharm installations
        installed = False
        for config_dir in pycharm_dirs:
            plugins_dir = config_dir / "plugins"
            plugins_dir.mkdir(exist_ok=True)

            dest = plugins_dir / "tb-language.jar"
            shutil.copy(plugin_jar, dest)
            print(f"  ✓ Installed to: {dest}")
            installed = True

        return installed

    def find_pycharm_config_dirs(self):
        """Find PyCharm config directories"""
        config_dirs = []
        home = Path.home()

        if self.system == "Windows":
            # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
            base = home / "AppData" / "Roaming" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        elif self.system == "Linux":
            # Linux: ~/.config/JetBrains/PyCharm*
            base = home / ".config" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

            # Also check old location
            old_base = home / ".PyCharm*"
            config_dirs.extend(home.glob(".PyCharm*"))

        elif self.system == "Darwin":
            # macOS: ~/Library/Application Support/JetBrains/PyCharm*
            base = home / "Library" / "Application Support" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        return [d for d in config_dirs if d.is_dir()]
build_executable()

Step 1: Build TB Language

Source code in toolboxv2/utils/tbx/install_support.py
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def build_executable(self):
    """Step 1: Build TB Language"""
    print("Step 1/4: Building TB Language...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
        "build"
    ])

    if result.returncode != 0:
        return False

    print("✓ Build successful")
    print()
    return True
build_pycharm_plugin()

Build PyCharm plugin JAR

Source code in toolboxv2/utils/tbx/install_support.py
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False
create_pycharm_plugin()

Create PyCharm plugin structure

Source code in toolboxv2/utils/tbx/install_support.py
199
200
201
202
203
204
205
206
207
208
def create_pycharm_plugin(self):
    """Create PyCharm plugin structure"""
    plugin_dir = self.root / "tb-lang-pycharm"
    plugin_dir.mkdir(exist_ok=True)

    # Create directory structure
    (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
    (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

    return True
find_pycharm_config_dirs()

Find PyCharm config directories

Source code in toolboxv2/utils/tbx/install_support.py
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
def find_pycharm_config_dirs(self):
    """Find PyCharm config directories"""
    config_dirs = []
    home = Path.home()

    if self.system == "Windows":
        # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
        base = home / "AppData" / "Roaming" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    elif self.system == "Linux":
        # Linux: ~/.config/JetBrains/PyCharm*
        base = home / ".config" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

        # Also check old location
        old_base = home / ".PyCharm*"
        config_dirs.extend(home.glob(".PyCharm*"))

    elif self.system == "Darwin":
        # macOS: ~/Library/Application Support/JetBrains/PyCharm*
        base = home / "Library" / "Application Support" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    return [d for d in config_dirs if d.is_dir()]
install_pycharm_plugin()

Install plugin to PyCharm

Source code in toolboxv2/utils/tbx/install_support.py
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
def install_pycharm_plugin(self):
    """Install plugin to PyCharm"""
    plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

    if not plugin_jar.exists():
        print("  Plugin JAR not found")
        return False

    # Find PyCharm config directory
    pycharm_dirs = self.find_pycharm_config_dirs()

    if not pycharm_dirs:
        print("  PyCharm installation not found")
        print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
        return False

    # Install to all found PyCharm installations
    installed = False
    for config_dir in pycharm_dirs:
        plugins_dir = config_dir / "plugins"
        plugins_dir.mkdir(exist_ok=True)

        dest = plugins_dir / "tb-language.jar"
        shutil.copy(plugin_jar, dest)
        print(f"  ✓ Installed to: {dest}")
        installed = True

    return installed
setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/install_support.py
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def setup_all(self):
    """Run complete setup"""
    print("═" * 70)
    print("  TB Language - Complete Setup")
    print("═" * 70)
    print()

    success = True

    # Step 1: Build
    if not self.build_executable():
        print("❌ Build failed!")
        return False

    # Step 2: System integration
    if not self.setup_system_integration():
        print("⚠️  System integration failed (optional)")
        success = False

    # Step 3: VS Code extension
    if not self.setup_vscode():
        print("⚠️  VS Code extension setup failed (optional)")
        success = False

    # Step 4: PyCharm plugin
    if not self.setup_pycharm():
        print("⚠️  PyCharm plugin setup failed (optional)")
        success = False

    print()
    print("═" * 70)
    if success:
        print("  ✓ Setup Complete!")
    else:
        print("  ⚠️  Setup completed with warnings")
    print("═" * 70)
    print()
    print("Next steps:")
    print("  1. Restart PyCharm and VS Code (if open)")
    print("  2. Create a test file: test.tbx")
    print("  3. Run it: tb run test.tbx")
    print("  4. Or double-click test.tbx to run")
    print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
    print()

    return success
setup_pycharm()

Step 4: PyCharm plugin

Source code in toolboxv2/utils/tbx/install_support.py
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
def setup_pycharm(self):
    """Step 4: PyCharm plugin"""
    print("Step 4/4: Installing PyCharm plugin...")
    print("-" * 70)

    pycharm_plugin = self.root / "tb-lang-pycharm"
    if not pycharm_plugin.exists():
        print("⚠️  PyCharm plugin directory not found")
        print("   Creating plugin structure...")
        if not self.create_pycharm_plugin():
            print()
            return False

    try:
        # Build plugin JAR
        print("  Building PyCharm plugin...")
        if not self.build_pycharm_plugin():
            print("⚠️  Plugin build failed")
            print()
            return False

        # Install to PyCharm
        print("  Installing to PyCharm...")
        if not self.install_pycharm_plugin():
            print("⚠️  Auto-install failed")
            print()
            return False

        print("✓ PyCharm plugin installed")
        print("  Please restart PyCharm to activate the plugin")
        print()
        return True

    except Exception as e:
        print(f"⚠️  Error: {e}")
        print()
        return False
setup_system_integration()

Step 2: System integration

Source code in toolboxv2/utils/tbx/install_support.py
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def setup_system_integration(self):
    """Step 2: System integration"""
    print("Step 2/4: Setting up system integration...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_setup.py"),
        "install"
    ])

    print()
    return result.returncode == 0
setup_vscode()

Step 3: VS Code extension

Source code in toolboxv2/utils/tbx/install_support.py
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def setup_vscode(self):
    """Step 3: VS Code extension"""
    print("Step 3/4: Installing VS Code extension...")
    print("-" * 70)

    vscode_ext = self.root / "tb-lang-vscode"
    if not vscode_ext.exists():
        print("⚠️  VS Code extension directory not found")
        print()
        return False

    try:
        # Check if npm is available
        subprocess.run(["npm", "--version"],
                       capture_output=True, check=True)

        # Install dependencies
        print("  Installing npm dependencies...")
        subprocess.run(["npm", "install"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Compile TypeScript
        print("  Compiling TypeScript...")
        subprocess.run(["npm", "run", "compile"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Try to install to VS Code
        print("  Installing to VS Code...")
        result = subprocess.run([
            "code", "--install-extension", str(vscode_ext.resolve())
        ], capture_output=True)

        if result.returncode == 0:
            print("✓ VS Code extension installed")
            print()
            return True
        else:
            print("⚠️  Could not auto-install to VS Code")
            print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
            print()
            return False

    except FileNotFoundError as e:
        print(f"⚠️  Tool not found: {e}")
        print("   npm: https://nodejs.org/")
        print("   VS Code: https://code.visualstudio.com/")
        print()
        return False
    except subprocess.CalledProcessError as e:
        print(f"⚠️  Command failed: {e}")
        print()
        return False
main()

Main entry point

Source code in toolboxv2/utils/tbx/install_support.py
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language Complete Setup"
    )
    parser.add_argument('--skip-build', action='store_true',
                        help='Skip building the executable')
    parser.add_argument('--skip-system', action='store_true',
                        help='Skip system integration')
    parser.add_argument('--skip-vscode', action='store_true',
                        help='Skip VS Code extension')
    parser.add_argument('--skip-pycharm', action='store_true',
                        help='Skip PyCharm plugin')
    parser.add_argument('--pycharm-only', action='store_true',
                        help='Only setup PyCharm plugin')

    args = parser.parse_args()

    setup = TBSetup()

    if args.pycharm_only:
        success = setup.setup_pycharm()
    else:
        # Full setup with skip options
        success = True

        if not args.skip_build:
            success = setup.build_executable() and success

        if not args.skip_system:
            setup.setup_system_integration()

        if not args.skip_vscode:
            setup.setup_vscode()

        if not args.skip_pycharm:
            setup.setup_pycharm()

    sys.exit(0 if success else 1)

proxy

ProxyUtil
Source code in toolboxv2/utils/proxy/prox_util.py
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
class ProxyUtil:
    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        # assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, timeout=6,
                        app: (App or AppType) | None = None,
                        remote_functions=None, peer=False, name='ProxyApp-client', do_connect=True, unix_socket=False,
                        test_override=False):
        self.class_instance = class_instance
        self.client = None
        self.test_override = test_override
        self.port = port
        self.host = host
        self.timeout = timeout
        if app is None:
            app = get_app("ProxyUtil")
        self.app = app
        self._name = name
        self.unix_socket = unix_socket
        if remote_functions is None:
            remote_functions = ["run_any", "a_run_any", "remove_mod", "save_load", "exit_main", "show_console", "hide_console",
                                "rrun_flow",
                                "get_autocompletion_dict",
                                "exit_main", "watch_mod"]
        self.remote_functions = remote_functions

        from toolboxv2.mods.SocketManager import SocketType
        self.connection_type = SocketType.client
        if peer:
            self.connection_type = SocketType.peer
        if do_connect:
            await self.connect()

    async def connect(self):
        client_result = await self.app.a_run_local(SOCKETMANAGER.CREATE_SOCKET,
                                           get_results=True,
                                           name=self._name,
                                           host=self.host,
                                           port=self.port,
                                           type_id=self.connection_type,
                                           max_connections=-1,
                                           return_full_object=True,
                                           test_override=self.test_override,
                                           unix_file=self.unix_socket)

        if client_result.is_error():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        if not client_result.is_data():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,
        result = await client_result.aget()
        if result is None or result.get('connection_error') != 0:
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        self.client = Result.ok(result)

    async def disconnect(self):
        time.sleep(1)
        close = self.client.get("close")
        await close()
        self.client = None

    async def reconnect(self):
        if self.client is not None:
            await self.disconnect()
        await self.connect()

    async def verify(self, message=b"verify"):
        await asyncio.sleep(1)
        # self.client.get('sender')({'keepalive': 0})
        await self.client.get('sender')(message)

    def __getattr__(self, name):

        # print(f"ProxyApp: {name}, {self.client is None}")
        if name == "on_exit":
            return self.disconnect
        if name == "rc":
            return self.reconnect

        if name == "r":
            try:
                return self.client.get('receiver_queue').get(timeout=self.timeout)
            except:
                return "No data"

        app_attr = getattr(self.class_instance, name)

        async def method(*args, **kwargs):
            # if name == 'run_any':
            #     print("method", name, kwargs.get('get_results', False), args[0])
            if self.client is None:
                await self.reconnect()
            if kwargs.get('spec', '-') == 'app':
                if asyncio.iscoroutinefunction(app_attr):
                    return await app_attr(*args, **kwargs)
                return app_attr(*args, **kwargs)
            try:
                if name in self.remote_functions:
                    if (name == 'run_any' or name == 'a_run_any') and not kwargs.get('get_results', False):
                        if asyncio.iscoroutinefunction(app_attr):
                            return await app_attr(*args, **kwargs)
                        return app_attr(*args, **kwargs)
                    if (name == 'run_any' or name == 'a_run_any') and kwargs.get('get_results', False):
                        if isinstance(args[0], Enum):
                            args = (args[0].__class__.NAME.value, args[0].value), args[1:]
                    self.app.sprint(f"Calling method {name}, {args=}, {kwargs}=")
                    await self.client.get('sender')({'name': name, 'args': args, 'kwargs': kwargs})
                    while Spinner("Waiting for result"):
                        try:
                            data = self.client.get('receiver_queue').get(timeout=self.timeout)
                            if isinstance(data, dict) and 'identifier' in data:
                                del data["identifier"]
                            if 'error' in data and 'origin' in data and 'result' in data and 'info' in data:
                                data = ApiResult(**data).as_result()
                            return data
                        except:
                            print("No data look later with class_instance.r")
                            return Result.default_internal_error("No data received from Demon."
                                                                 " uns class_instance.r to get data later")
            except:
                if self.client.get('socket') is None:
                    self.client = None
            return app_attr(*args, **kwargs)

        if callable(app_attr) and name in self.remote_functions and self.client is not None:
            return method
        return app_attr
__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/proxy/prox_util.py
20
21
22
23
24
25
26
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.async_initialized = False
__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/proxy/prox_util.py
28
29
30
31
32
33
34
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    # assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self
prox_util
ProxyUtil
Source code in toolboxv2/utils/proxy/prox_util.py
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
class ProxyUtil:
    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.async_initialized = False

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        # assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()

    async def __ainit__(self, class_instance: Any, host='0.0.0.0', port=6587, timeout=6,
                        app: (App or AppType) | None = None,
                        remote_functions=None, peer=False, name='ProxyApp-client', do_connect=True, unix_socket=False,
                        test_override=False):
        self.class_instance = class_instance
        self.client = None
        self.test_override = test_override
        self.port = port
        self.host = host
        self.timeout = timeout
        if app is None:
            app = get_app("ProxyUtil")
        self.app = app
        self._name = name
        self.unix_socket = unix_socket
        if remote_functions is None:
            remote_functions = ["run_any", "a_run_any", "remove_mod", "save_load", "exit_main", "show_console", "hide_console",
                                "rrun_flow",
                                "get_autocompletion_dict",
                                "exit_main", "watch_mod"]
        self.remote_functions = remote_functions

        from toolboxv2.mods.SocketManager import SocketType
        self.connection_type = SocketType.client
        if peer:
            self.connection_type = SocketType.peer
        if do_connect:
            await self.connect()

    async def connect(self):
        client_result = await self.app.a_run_local(SOCKETMANAGER.CREATE_SOCKET,
                                           get_results=True,
                                           name=self._name,
                                           host=self.host,
                                           port=self.port,
                                           type_id=self.connection_type,
                                           max_connections=-1,
                                           return_full_object=True,
                                           test_override=self.test_override,
                                           unix_file=self.unix_socket)

        if client_result.is_error():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        if not client_result.is_data():
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        # 'socket': socket,
        # 'receiver_socket': r_socket,
        # 'host': host,
        # 'port': port,
        # 'p2p-port': endpoint_port,
        # 'sender': send,
        # 'receiver_queue': receiver_queue,
        # 'connection_error': connection_error,
        # 'receiver_thread': s_thread,
        # 'keepalive_thread': keep_alive_thread,
        # 'running_dict': running_dict,
        # 'client_to_receiver_thread': to_receive,
        # 'client_receiver_threads': threeds,
        result = await client_result.aget()
        if result is None or result.get('connection_error') != 0:
            raise Exception(f"Client {self._name} error: {client_result.print(False)}")
        self.client = Result.ok(result)

    async def disconnect(self):
        time.sleep(1)
        close = self.client.get("close")
        await close()
        self.client = None

    async def reconnect(self):
        if self.client is not None:
            await self.disconnect()
        await self.connect()

    async def verify(self, message=b"verify"):
        await asyncio.sleep(1)
        # self.client.get('sender')({'keepalive': 0})
        await self.client.get('sender')(message)

    def __getattr__(self, name):

        # print(f"ProxyApp: {name}, {self.client is None}")
        if name == "on_exit":
            return self.disconnect
        if name == "rc":
            return self.reconnect

        if name == "r":
            try:
                return self.client.get('receiver_queue').get(timeout=self.timeout)
            except:
                return "No data"

        app_attr = getattr(self.class_instance, name)

        async def method(*args, **kwargs):
            # if name == 'run_any':
            #     print("method", name, kwargs.get('get_results', False), args[0])
            if self.client is None:
                await self.reconnect()
            if kwargs.get('spec', '-') == 'app':
                if asyncio.iscoroutinefunction(app_attr):
                    return await app_attr(*args, **kwargs)
                return app_attr(*args, **kwargs)
            try:
                if name in self.remote_functions:
                    if (name == 'run_any' or name == 'a_run_any') and not kwargs.get('get_results', False):
                        if asyncio.iscoroutinefunction(app_attr):
                            return await app_attr(*args, **kwargs)
                        return app_attr(*args, **kwargs)
                    if (name == 'run_any' or name == 'a_run_any') and kwargs.get('get_results', False):
                        if isinstance(args[0], Enum):
                            args = (args[0].__class__.NAME.value, args[0].value), args[1:]
                    self.app.sprint(f"Calling method {name}, {args=}, {kwargs}=")
                    await self.client.get('sender')({'name': name, 'args': args, 'kwargs': kwargs})
                    while Spinner("Waiting for result"):
                        try:
                            data = self.client.get('receiver_queue').get(timeout=self.timeout)
                            if isinstance(data, dict) and 'identifier' in data:
                                del data["identifier"]
                            if 'error' in data and 'origin' in data and 'result' in data and 'info' in data:
                                data = ApiResult(**data).as_result()
                            return data
                        except:
                            print("No data look later with class_instance.r")
                            return Result.default_internal_error("No data received from Demon."
                                                                 " uns class_instance.r to get data later")
            except:
                if self.client.get('socket') is None:
                    self.client = None
            return app_attr(*args, **kwargs)

        if callable(app_attr) and name in self.remote_functions and self.client is not None:
            return method
        return app_attr
__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/proxy/prox_util.py
20
21
22
23
24
25
26
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.async_initialized = False
__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/proxy/prox_util.py
28
29
30
31
32
33
34
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    # assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self

security

Code
Source code in toolboxv2/utils/security/cryp.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()
decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name Type Description Default
encrypted_text_hex str

Der verschlüsselte Text als Hex-String.

required
private_key_str str

Der private Schlüssel als String.

required

Returns:

Name Type Description
str str

Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"
decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
encrypted_text str

Der zu entschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required
to_str bool

default true returns str if false returns bytes

True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()
encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
public_key_str str

Der öffentliche Schlüssel als String oder im pem format.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"
encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"
generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name Type Description Default
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key
generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name Type Description Default
length int

Die Länge der zu generierenden Zeichenkette.

required

Returns:

Name Type Description
str str

Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py
81
82
83
84
85
86
87
88
89
90
91
92
@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)
generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name Type Description
int int

Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py
114
115
116
117
118
119
120
121
122
@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)
generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name Type Description
str str or bytes

Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
140
141
142
143
144
145
146
147
148
149
150
151
@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key
load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name Type Description Default
directory str

Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

'keys'

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type Description
FileNotFoundError

Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key
one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name Type Description Default
text str

Der zu hashende Text.

required
salt str

Der Salt-Wert.

''
pepper str

Der Pepper-Wert.

''
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Name Type Description
str str

Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()
pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name Type Description Default
pem_key str

Der PEM-kodierte öffentliche Schlüssel.

required

Returns:

Name Type Description
PublicKey

Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py
435
436
437
438
439
440
441
442
443
444
445
446
447
@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key
public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name Type Description Default
public_key PublicKey

Das PublicKey-Objekt.

required

Returns:

Name Type Description
str

Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()
save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name Type Description Default
public_key str

Der öffentliche Schlüssel im PEM-Format

required
private_key str

Der private Schlüssel im PEM-Format

required
directory str

Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen

'keys'
Source code in toolboxv2/utils/security/cryp.py
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)
cryp
Code
Source code in toolboxv2/utils/security/cryp.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()
decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name Type Description Default
encrypted_text_hex str

Der verschlüsselte Text als Hex-String.

required
private_key_str str

Der private Schlüssel als String.

required

Returns:

Name Type Description
str str

Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"
decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
encrypted_text str

Der zu entschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required
to_str bool

default true returns str if false returns bytes

True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()
encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
public_key_str str

Der öffentliche Schlüssel als String oder im pem format.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"
encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"
generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name Type Description Default
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key
generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name Type Description Default
length int

Die Länge der zu generierenden Zeichenkette.

required

Returns:

Name Type Description
str str

Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py
81
82
83
84
85
86
87
88
89
90
91
92
@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)
generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name Type Description
int int

Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py
114
115
116
117
118
119
120
121
122
@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)
generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name Type Description
str str or bytes

Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
140
141
142
143
144
145
146
147
148
149
150
151
@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key
load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name Type Description Default
directory str

Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

'keys'

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type Description
FileNotFoundError

Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key
one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name Type Description Default
text str

Der zu hashende Text.

required
salt str

Der Salt-Wert.

''
pepper str

Der Pepper-Wert.

''
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Name Type Description
str str

Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()
pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name Type Description Default
pem_key str

Der PEM-kodierte öffentliche Schlüssel.

required

Returns:

Name Type Description
PublicKey

Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py
435
436
437
438
439
440
441
442
443
444
445
446
447
@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key
public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name Type Description Default
public_key PublicKey

Das PublicKey-Objekt.

required

Returns:

Name Type Description
str

Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()
save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name Type Description Default
public_key str

Der öffentliche Schlüssel im PEM-Format

required
private_key str

Der private Schlüssel im PEM-Format

required
directory str

Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen

'keys'
Source code in toolboxv2/utils/security/cryp.py
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)

singelton_class

Singleton

Singleton metaclass for ensuring only one instance of a class.

Source code in toolboxv2/utils/singelton_class.py
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
class Singleton(type):
    """
    Singleton metaclass for ensuring only one instance of a class.
    """

    _instances = {}
    _kwargs = {}
    _args = {}

    def __call__(cls, *args, **kwargs):
        if cls not in cls._instances:
            cls._instances[cls] = super().__call__(*args, **kwargs)
            cls._args[cls] = args
            cls._kwargs[cls] = kwargs
        return cls._instances[cls]

system

AppType
Source code in toolboxv2/utils/system/types.py
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
class AppType:
    prefix: str
    id: str
    globals: dict[str, Any] = {"root": dict, }
    locals: dict[str, Any] = {"user": {'app': "self"}, }

    local_test: bool = False
    start_dir: str
    data_dir: str
    config_dir: str
    info_dir: str
    is_server:bool = False

    logger: logging.Logger
    logging_filename: str

    api_allowed_mods_list: list[str] = []

    version: str
    loop: asyncio.AbstractEventLoop

    keys: dict[str, str] = {
        "MACRO": "macro~~~~:",
        "MACRO_C": "m_color~~:",
        "HELPER": "helper~~~:",
        "debug": "debug~~~~:",
        "id": "name-spa~:",
        "st-load": "mute~load:",
        "comm-his": "comm-his~:",
        "develop-mode": "dev~mode~:",
        "provider::": "provider::",
    }

    defaults: dict[str, (bool or dict or dict[str, dict[str, str]] or str or list[str] or list[list]) | None] = {
        "MACRO": list[str],
        "MACRO_C": dict,
        "HELPER": dict,
        "debug": str,
        "id": str,
        "st-load": False,
        "comm-his": list[list],
        "develop-mode": bool,
    }

    cluster_manager: ClusterManager
    root_blob_storage: BlobStorage
    config_fh: FileHandler
    _debug: bool
    flows: dict[str, Callable]
    dev_modi: bool
    functions: dict[str, Any]
    modules: dict[str, Any]

    interface_type: ToolBoxInterfaces
    REFIX: str

    alive: bool
    called_exit: tuple[bool, float]
    args_sto: AppArgs
    system_flag = None
    session = None
    appdata = None
    exit_tasks = []

    enable_profiling: bool = False
    sto = None

    websocket_handlers: dict[str, dict[str, Callable]] = {}
    _rust_ws_bridge: Any = None

    docs_reader: Callable | None = None
    docs_writer: Callable | None = None
    get_update_suggestions: Callable | None = None
    auto_update_docs: Callable | None = None
    source_code_lookup: Callable | None = None

    initial_docs_parse: Callable | None = None

    def __init__(self, prefix=None, args=None):
        self.args_sto = args
        self.prefix = prefix
        self._footprint_start_time = time.time()
        self._process = psutil.Process(os.getpid())

        # Tracking-Daten für Min/Max/Avg
        self._footprint_metrics = {
            'memory': {'max': 0, 'min': float('inf'), 'samples': []},
            'cpu': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_read': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_write': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_sent': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_recv': {'max': 0, 'min': float('inf'), 'samples': []},
        }

        # Initial Disk/Network Counters
        try:
            io_counters = self._process.io_counters()
            self._initial_disk_read = io_counters.read_bytes
            self._initial_disk_write = io_counters.write_bytes
        except (AttributeError, OSError):
            self._initial_disk_read = 0
            self._initial_disk_write = 0

        try:
            net_io = psutil.net_io_counters()
            self._initial_network_sent = net_io.bytes_sent
            self._initial_network_recv = net_io.bytes_recv
        except (AttributeError, OSError):
            self._initial_network_sent = 0
            self._initial_network_recv = 0

    def _update_metric_tracking(self, metric_name: str, value: float):
        """Aktualisiert Min/Max/Avg für eine Metrik"""
        metrics = self._footprint_metrics[metric_name]
        metrics['max'] = max(metrics['max'], value)
        metrics['min'] = min(metrics['min'], value)
        metrics['samples'].append(value)

        # Begrenze die Anzahl der Samples (letzte 1000)
        if len(metrics['samples']) > 1000:
            metrics['samples'] = metrics['samples'][-1000:]

    def _get_metric_avg(self, metric_name: str) -> float:
        """Berechnet Durchschnitt einer Metrik"""
        samples = self._footprint_metrics[metric_name]['samples']
        return sum(samples) / len(samples) if samples else 0

    def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
        """
        Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

        Args:
            update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

        Returns:
            FootprintMetrics mit allen erfassten Metriken
        """
        current_time = time.time()
        uptime_seconds = current_time - self._footprint_start_time

        # Formatierte Uptime
        uptime_delta = timedelta(seconds=int(uptime_seconds))
        uptime_formatted = str(uptime_delta)

        # Memory Metrics (in MB)
        try:
            mem_info = self._process.memory_info()
            memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
            memory_percent = self._process.memory_percent()

            if update_tracking:
                self._update_metric_tracking('memory', memory_current)

            memory_max = self._footprint_metrics['memory']['max']
            memory_min = self._footprint_metrics['memory']['min']
            if memory_min == float('inf'):
                memory_min = memory_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            memory_current = memory_max = memory_min = memory_percent = 0

        # CPU Metrics
        try:
            cpu_percent_current = self._process.cpu_percent(interval=0.1)
            cpu_times = self._process.cpu_times()
            cpu_time_seconds = cpu_times.user + cpu_times.system

            if update_tracking:
                self._update_metric_tracking('cpu', cpu_percent_current)

            cpu_percent_max = self._footprint_metrics['cpu']['max']
            cpu_percent_min = self._footprint_metrics['cpu']['min']
            cpu_percent_avg = self._get_metric_avg('cpu')

            if cpu_percent_min == float('inf'):
                cpu_percent_min = cpu_percent_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            cpu_percent_current = cpu_percent_max = 0
            cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

        # Disk I/O Metrics (in MB)
        try:
            io_counters = self._process.io_counters()
            disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
            disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

            disk_read_mb = disk_read_bytes / (1024 * 1024)
            disk_write_mb = disk_write_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('disk_read', disk_read_mb)
                self._update_metric_tracking('disk_write', disk_write_mb)

            disk_read_max = self._footprint_metrics['disk_read']['max']
            disk_read_min = self._footprint_metrics['disk_read']['min']
            disk_write_max = self._footprint_metrics['disk_write']['max']
            disk_write_min = self._footprint_metrics['disk_write']['min']

            if disk_read_min == float('inf'):
                disk_read_min = disk_read_mb
            if disk_write_min == float('inf'):
                disk_write_min = disk_write_mb
        except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
            disk_read_mb = disk_write_mb = 0
            disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

        # Network I/O Metrics (in MB)
        try:
            net_io = psutil.net_io_counters()
            network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
            network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

            network_sent_mb = network_sent_bytes / (1024 * 1024)
            network_recv_mb = network_recv_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('network_sent', network_sent_mb)
                self._update_metric_tracking('network_recv', network_recv_mb)

            network_sent_max = self._footprint_metrics['network_sent']['max']
            network_sent_min = self._footprint_metrics['network_sent']['min']
            network_recv_max = self._footprint_metrics['network_recv']['max']
            network_recv_min = self._footprint_metrics['network_recv']['min']

            if network_sent_min == float('inf'):
                network_sent_min = network_sent_mb
            if network_recv_min == float('inf'):
                network_recv_min = network_recv_mb
        except (AttributeError, OSError):
            network_sent_mb = network_recv_mb = 0
            network_sent_max = network_sent_min = 0
            network_recv_max = network_recv_min = 0

        # Process Info
        try:
            process_id = self._process.pid
            threads = self._process.num_threads()
            open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
            connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

            open_files = len(open_files_path)
            connections = len(connections_uri)
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            process_id = os.getpid()
            threads = open_files = connections = 0
            open_files_path = []
            connections_uri = []

        return FootprintMetrics(
            start_time=self._footprint_start_time,
            uptime_seconds=uptime_seconds,
            uptime_formatted=uptime_formatted,
            memory_current=memory_current,
            memory_max=memory_max,
            memory_min=memory_min,
            memory_percent=memory_percent,
            cpu_percent_current=cpu_percent_current,
            cpu_percent_max=cpu_percent_max,
            cpu_percent_min=cpu_percent_min,
            cpu_percent_avg=cpu_percent_avg,
            cpu_time_seconds=cpu_time_seconds,
            disk_read_mb=disk_read_mb,
            disk_write_mb=disk_write_mb,
            disk_read_max=disk_read_max,
            disk_read_min=disk_read_min,
            disk_write_max=disk_write_max,
            disk_write_min=disk_write_min,
            network_sent_mb=network_sent_mb,
            network_recv_mb=network_recv_mb,
            network_sent_max=network_sent_max,
            network_sent_min=network_sent_min,
            network_recv_max=network_recv_max,
            network_recv_min=network_recv_min,
            process_id=process_id,
            threads=threads,
            open_files=open_files,
            connections=connections,
            open_files_path=open_files_path,
            connections_uri=connections_uri,
        )

    def print_footprint(self, detailed: bool = True) -> str:
        """
        Gibt den Footprint formatiert aus.

        Args:
            detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

        Returns:
            Formatierter Footprint-String
        """
        metrics = self.footprint()

        output = [
            "=" * 70,
            f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
            "=" * 70,
            f"\n📊 UPTIME",
            f"  Runtime: {metrics.uptime_formatted}",
            f"  Seconds: {metrics.uptime_seconds:.2f}s",
            f"\n💾 MEMORY USAGE",
            f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
            f"  Maximum:  {metrics.memory_max:.2f} MB",
            f"  Minimum:  {metrics.memory_min:.2f} MB",
        ]

        if detailed:
            helper_ = '\n\t- '.join(metrics.open_files_path)
            helper__ = '\n\t- '.join(metrics.connections_uri)
            output.extend([
                f"\n⚙️  CPU USAGE",
                f"  Current:  {metrics.cpu_percent_current:.2f}%",
                f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
                f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
                f"  Average:  {metrics.cpu_percent_avg:.2f}%",
                f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
                f"\n💿 DISK I/O",
                f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
                f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
                f"\n🌐 NETWORK I/O",
                f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
                f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
                f"\n🔧 PROCESS INFO",
                f"  PID:         {metrics.process_id}",
                f"  Threads:     {metrics.threads}",
                f"\n📂 OPEN FILES",
                f"  Open Files:  {metrics.open_files}",
                f"  Open Files Path: \n\t- {helper_}",
                f"\n🔗 NETWORK CONNECTIONS",
                f"  Connections: {metrics.connections}",
                f"  Connections URI: \n\t- {helper__}",
            ])

        output.append("=" * 70)

        return "\n".join(output)



    def start_server(self):
        from toolboxv2.utils.clis.api import manage_server
        if self.is_server:
            return
        manage_server("start")
        self.is_server = False

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        """proxi attr"""

    @property
    def debug(self):
        """proxi attr"""
        return self._debug

    def debug_rains(self, e):
        """proxi attr"""

    def set_flows(self, r):
        """proxi attr"""

    async def run_flows(self, name, **kwargs):
        """proxi attr"""

    def rrun_flows(self, name, **kwargs):
        """proxi attr"""

    def idle(self):
        import time
        self.print("idle")
        try:
            while self.alive:
                time.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("idle done")

    async def a_idle(self):
        self.print("a idle (running :"+("online)" if hasattr(self, 'daemon_app') else "offline)"))
        try:
            if hasattr(self, 'daemon_app'):
                await self.daemon_app.connect(self)
            else:
                while self.alive:
                    await asyncio.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("a idle done")

    @debug.setter
    def debug(self, value):
        """proxi attr"""

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):
        """proxi attr"""

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        """proxi attr"""

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        """proxi attr"""

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
        """proxi attr"""

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
        """proxi attr"""

    def save_initialized_module(self, tools_class, spec):
        """proxi attr"""

    def mod_online(self, mod_name, installed=False):
        """proxi attr"""

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0):
        """proxi attr"""

    def save_exit(self):
        """proxi attr"""

    def load_mod(self, mod_name: str, mlm='I', **kwargs):
        """proxi attr"""

    async def init_module(self, modular):
        return await self.load_mod(modular)

    async def load_external_mods(self):
        """proxi attr"""

    async def load_all_mods_in_file(self, working_dir="mods"):
        """proxi attr"""

    def get_all_mods(self, working_dir="mods", path_to="./runtime"):
        """proxi attr"""

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    def print_ok(self):
        """proxi attr"""
        self.logger.info("OK")

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        """proxi attr"""

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
        """proxi attr"""

    def remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    def exit(self):
        """proxi attr"""

    def web_context(self) -> str:
        """returns the build index ( toolbox web component )"""

    async def a_exit(self):
        """proxi attr"""

    def save_load(self, modname, spec='app'):
        """proxi attr"""

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """

    def run_a_from_sync(self, function, *args):
        """
        run a async fuction
        """

    def run_bg_task_advanced(self, task, *args, **kwargs):
        """
        proxi attr
        """

    def wait_for_bg_tasks(self, timeout=None):
        """
        proxi attr
        """

    def run_bg_task(self, task):
        """
                run a async fuction
                """
    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        """proxi attr"""

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        """proxi attr"""

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                       args_=None,
                       kwargs_=None,
                       *args, **kwargs):
        """run a function remote via http / https"""

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):
        """proxi attr"""

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):
        """proxi attr"""

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        """proxi attr"""

    @staticmethod
    def print(text, *args, **kwargs):
        """proxi attr"""

    @staticmethod
    def sprint(text, *args, **kwargs):
        """proxi attr"""

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def _register_function(self, module_name, func_name, data):
        """proxi attr"""

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial=False,
                          exit_f=False,
                          test=True,
                          samples=None,
                          state=None,
                          pre_compute=None,
                          post_compute=None,
                          memory_cache=False,
                          file_cache=False,
                          row=False,
                          request_as_kwarg=False,
                          memory_cache_max_size=100,
                          memory_cache_ttl=300,
                          websocket_handler: str | None = None,):
        """proxi attr"""

        # data = {
        #     "type": type_,
        #     "module_name": module_name,
        #     "func_name": func_name,
        #     "level": level,
        #     "restrict_in_virtual_mode": restrict_in_virtual_mode,
        #     "func": func,
        #     "api": api,
        #     "helper": helper,
        #     "version": version,
        #     "initial": initial,
        #     "exit_f": exit_f,
        #     "__module__": func.__module__,
        #     "signature": sig,
        #     "params": params,
        #     "state": (
        #         False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
        #     "do_test": test,
        #     "samples": samples,
        #     "request_as_kwarg": request_as_kwarg,

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str or None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           row=False,
           request_as_kwarg: bool = False,
           state: bool or None = None,
           level: int = 0,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      row=row,
                                      request_as_kwarg=request_as_kwarg,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl)

    def print_functions(self, name=None):


        if not self.functions:
            return

        def helper(_functions):
            for func_name, data in _functions.items():
                if not isinstance(data, dict):
                    continue

                func_type = data.get('type', 'Unknown')
                func_level = 'r' if data['level'] == -1 else data['level']
                api_status = 'Api' if data.get('api', False) else 'Non-Api'

                print(f"  Function: {func_name}{data.get('signature', '()')}; "
                      f"Type: {func_type}, Level: {func_level}, {api_status}")

        if name is not None:
            functions = self.functions.get(name)
            if functions is not None:
                print(f"\nModule: {name}; Type: {functions.get('app_instance_type', 'Unknown')}")
                helper(functions)
                return
        for module, functions in self.functions.items():
            print(f"\nModule: {module}; Type: {functions.get('app_instance_type', 'Unknown')}")
            helper(functions)

    def save_autocompletion_dict(self):
        """proxi attr"""

    def get_autocompletion_dict(self):
        """proxi attr"""

    def get_username(self, get_input=False, default="loot") -> str:
        """proxi attr"""

    def save_registry_as_enums(self, directory: str, filename: str):
        """proxi attr"""

    async def execute_all_functions_(self, m_query='', f_query=''):
        print("Executing all functions")
        from ..extras import generate_test_cases
        all_data = {
            "modular_run": 0,
            "modular_fatal_error": 0,
            "errors": 0,
            "modular_sug": 0,
            "coverage": [],
            "total_coverage": {},
        }
        items = list(self.functions.items()).copy()
        for module_name, functions in items:
            infos = {
                "functions_run": 0,
                "functions_fatal_error": 0,
                "error": 0,
                "functions_sug": 0,
                'calls': {},
                'callse': {},
                "coverage": [0, 0],
            }
            all_data['modular_run'] += 1
            if not module_name.startswith(m_query):
                all_data['modular_sug'] += 1
                continue

            with Spinner(message=f"In {module_name}| "):
                f_items = list(functions.items()).copy()
                for function_name, function_data in f_items:
                    if not isinstance(function_data, dict):
                        continue
                    if not function_name.startswith(f_query):
                        continue
                    test: list = function_data.get('do_test')
                    # print(test, module_name, function_name, function_data)
                    infos["coverage"][0] += 1
                    if test is False:
                        continue

                    with Spinner(message=f"\t\t\t\t\t\tfuction {function_name}..."):
                        params: list = function_data.get('params')
                        sig: signature = function_data.get('signature')
                        state: bool = function_data.get('state')
                        samples: bool = function_data.get('samples')

                        test_kwargs_list = [{}]

                        if params is not None:
                            test_kwargs_list = samples if samples is not None else generate_test_cases(sig=sig)
                            # print(test_kwargs)
                            # print(test_kwargs[0])
                            # test_kwargs = test_kwargs_list[0]
                        # print(module_name, function_name, test_kwargs_list)
                        infos["coverage"][1] += 1
                        for test_kwargs in test_kwargs_list:
                            try:
                                # print(f"test Running {state=} |{module_name}.{function_name}")
                                result = await self.a_run_function((module_name, function_name),
                                                                   tb_run_function_with_state=state,
                                                                   **test_kwargs)
                                if not isinstance(result, Result):
                                    result = Result.ok(result)
                                if result.info.exec_code == 0:
                                    infos['calls'][function_name] = [test_kwargs, str(result)]
                                    infos['functions_sug'] += 1
                                else:
                                    infos['functions_sug'] += 1
                                    infos['error'] += 1
                                    infos['callse'][function_name] = [test_kwargs, str(result)]
                            except Exception as e:
                                infos['functions_fatal_error'] += 1
                                infos['callse'][function_name] = [test_kwargs, str(e)]
                            finally:
                                infos['functions_run'] += 1

                if infos['functions_run'] == infos['functions_sug']:
                    all_data['modular_sug'] += 1
                else:
                    all_data['modular_fatal_error'] += 1
                if infos['error'] > 0:
                    all_data['errors'] += infos['error']

                all_data[module_name] = infos
                if infos['coverage'][0] == 0:
                    c = 0
                else:
                    c = infos['coverage'][1] / infos['coverage'][0]
                all_data["coverage"].append(f"{module_name}:{c:.2f}\n")
        total_coverage = sum([float(t.split(":")[-1]) for t in all_data["coverage"]]) / len(all_data["coverage"])
        print(
            f"\n{all_data['modular_run']=}\n{all_data['modular_sug']=}\n{all_data['modular_fatal_error']=}\n{total_coverage=}")
        d = analyze_data(all_data)
        return Result.ok(data=all_data, data_info=d)

    async def execute_function_test(self, module_name: str, function_name: str,
                                    function_data: dict, test_kwargs: dict,
                                    profiler: cProfile.Profile) -> tuple[bool, str, dict, float]:
        start_time = time.time()
        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            try:
                result = await self.a_run_function(
                    (module_name, function_name),
                    tb_run_function_with_state=function_data.get('state'),
                    **test_kwargs
                )

                if not isinstance(result, Result):
                    result = Result.ok(result)

                success = result.info.exec_code == 0
                execution_time = time.time() - start_time
                return success, str(result), test_kwargs, execution_time
            except Exception as e:
                execution_time = time.time() - start_time
                return False, str(e), test_kwargs, execution_time

    async def process_function(self, module_name: str, function_name: str,
                               function_data: dict, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()
        info = ModuleInfo()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            if not isinstance(function_data, dict):
                return function_name, info

            test = function_data.get('do_test')
            info.coverage[0] += 1

            if test is False:
                return function_name, info

            params = function_data.get('params')
            sig = function_data.get('signature')
            samples = function_data.get('samples')

            test_kwargs_list = [{}] if params is None else (
                samples if samples is not None else generate_test_cases(sig=sig)
            )

            info.coverage[1] += 1

            # Create tasks for all test cases
            tasks = [
                self.execute_function_test(module_name, function_name, function_data, test_kwargs, profiler)
                for test_kwargs in test_kwargs_list
            ]

            # Execute all tests concurrently
            results = await asyncio.gather(*tasks)

            total_execution_time = 0
            for success, result_str, test_kwargs, execution_time in results:
                info.functions_run += 1
                total_execution_time += execution_time

                if success:
                    info.functions_sug += 1
                    info.calls[function_name] = [test_kwargs, result_str]
                else:
                    info.functions_sug += 1
                    info.error += 1
                    info.callse[function_name] = [test_kwargs, result_str]

            info.execution_time = time.time() - start_time
            return function_name, info

    async def process_module(self, module_name: str, functions: dict,
                             f_query: str, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_function(module_name, fname, fdata, profiler)
                    for fname, fdata in functions.items()
                    if fname.startswith(f_query)
                ]

                if not tasks:
                    return module_name, ModuleInfo()

                results = await asyncio.gather(*tasks)

                # Combine results from all functions in the module
                combined_info = ModuleInfo()
                total_execution_time = 0

                for _, info in results:
                    combined_info.functions_run += info.functions_run
                    combined_info.functions_fatal_error += info.functions_fatal_error
                    combined_info.error += info.error
                    combined_info.functions_sug += info.functions_sug
                    combined_info.calls.update(info.calls)
                    combined_info.callse.update(info.callse)
                    combined_info.coverage[0] += info.coverage[0]
                    combined_info.coverage[1] += info.coverage[1]
                    total_execution_time += info.execution_time

                combined_info.execution_time = time.time() - start_time
                return module_name, combined_info

    async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
        """
        Execute all functions with parallel processing and optional profiling.

        Args:
            m_query (str): Module name query filter
            f_query (str): Function name query filter
            enable_profiling (bool): Enable detailed profiling information
        """
        print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

        start_time = time.time()
        stats = ExecutionStats()
        items = list(self.functions.items()).copy()

        # Set up profiling
        self.enable_profiling = enable_profiling
        profiler = cProfile.Profile()

        with profile_section(profiler, enable_profiling):
            # Filter modules based on query
            filtered_modules = [
                (mname, mfuncs) for mname, mfuncs in items
                if mname.startswith(m_query)
            ]

            stats.modular_run = len(filtered_modules)

            # Process all modules concurrently
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_module(mname, mfuncs, f_query, profiler)
                    for mname, mfuncs in filtered_modules
                ]

                results = await asyncio.gather(*tasks)

            # Combine results and calculate statistics
            for module_name, info in results:
                if info.functions_run == info.functions_sug:
                    stats.modular_sug += 1
                else:
                    stats.modular_fatal_error += 1

                stats.errors += info.error

                # Calculate coverage
                coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
                stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

                # Store module info
                stats.__dict__[module_name] = info

            # Calculate total coverage
            total_coverage = (
                sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
                if stats.coverage else 0
            )

            stats.total_execution_time = time.time() - start_time

            # Generate profiling stats if enabled
            if enable_profiling:
                s = io.StringIO()
                ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
                ps.print_stats()
                stats.profiling_data = {
                    'detailed_stats': s.getvalue(),
                    'total_time': stats.total_execution_time,
                    'function_count': stats.modular_run,
                    'successful_functions': stats.modular_sug
                }

            print(
                f"\n{stats.modular_run=}"
                f"\n{stats.modular_sug=}"
                f"\n{stats.modular_fatal_error=}"
                f"\n{total_coverage=}"
                f"\nTotal execution time: {stats.total_execution_time:.2f}s"
            )

            if enable_profiling:
                print("\nProfiling Summary:")
                print(f"{'=' * 50}")
                print("Top 10 time-consuming functions:")
                ps.print_stats(10)

            analyzed_data = analyze_data(stats.__dict__)
            return Result.ok(data=stats.__dict__, data_info=analyzed_data)
debug property writable

proxi attr

a_exit() async

proxi attr

Source code in toolboxv2/utils/system/types.py
1903
1904
async def a_exit(self):
    """proxi attr"""
a_fuction_runner(function, function_data, args, kwargs) async

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py
1968
1969
1970
1971
1972
1973
1974
1975
1976
async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """
a_remove_mod(mod_name, spec='app', delete=True) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1894
1895
async def a_remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""
a_run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1990
1991
1992
1993
1994
1995
async def a_run_any(self, mod_function_name: Enum or str or tuple,
                    backwords_compability_variabel_string_holder=None,
                    get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                    kwargs_=None,
                    *args, **kwargs):
    """proxi attr"""
a_run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1948
1949
1950
1951
1952
1953
1954
1955
1956
async def a_run_function(self, mod_function_name: Enum or tuple,
                         tb_run_function_with_state=True,
                         tb_run_with_specification='app',
                         args_=None,
                         kwargs_=None,
                         *args,
                         **kwargs) -> Result:

    """proxi attr"""
debug_rains(e)

proxi attr

Source code in toolboxv2/utils/system/types.py
1787
1788
def debug_rains(self, e):
    """proxi attr"""
disconnect(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1775
1776
1777
@staticmethod
async def disconnect(*args, **kwargs):
    """proxi attr"""
execute_all_functions(m_query='', f_query='', enable_profiling=True) async

Execute all functions with parallel processing and optional profiling.

Parameters:

Name Type Description Default
m_query str

Module name query filter

''
f_query str

Function name query filter

''
enable_profiling bool

Enable detailed profiling information

True
Source code in toolboxv2/utils/system/types.py
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
    """
    Execute all functions with parallel processing and optional profiling.

    Args:
        m_query (str): Module name query filter
        f_query (str): Function name query filter
        enable_profiling (bool): Enable detailed profiling information
    """
    print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

    start_time = time.time()
    stats = ExecutionStats()
    items = list(self.functions.items()).copy()

    # Set up profiling
    self.enable_profiling = enable_profiling
    profiler = cProfile.Profile()

    with profile_section(profiler, enable_profiling):
        # Filter modules based on query
        filtered_modules = [
            (mname, mfuncs) for mname, mfuncs in items
            if mname.startswith(m_query)
        ]

        stats.modular_run = len(filtered_modules)

        # Process all modules concurrently
        async with asyncio.Semaphore(mp.cpu_count()):
            tasks = [
                self.process_module(mname, mfuncs, f_query, profiler)
                for mname, mfuncs in filtered_modules
            ]

            results = await asyncio.gather(*tasks)

        # Combine results and calculate statistics
        for module_name, info in results:
            if info.functions_run == info.functions_sug:
                stats.modular_sug += 1
            else:
                stats.modular_fatal_error += 1

            stats.errors += info.error

            # Calculate coverage
            coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
            stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

            # Store module info
            stats.__dict__[module_name] = info

        # Calculate total coverage
        total_coverage = (
            sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
            if stats.coverage else 0
        )

        stats.total_execution_time = time.time() - start_time

        # Generate profiling stats if enabled
        if enable_profiling:
            s = io.StringIO()
            ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
            ps.print_stats()
            stats.profiling_data = {
                'detailed_stats': s.getvalue(),
                'total_time': stats.total_execution_time,
                'function_count': stats.modular_run,
                'successful_functions': stats.modular_sug
            }

        print(
            f"\n{stats.modular_run=}"
            f"\n{stats.modular_sug=}"
            f"\n{stats.modular_fatal_error=}"
            f"\n{total_coverage=}"
            f"\nTotal execution time: {stats.total_execution_time:.2f}s"
        )

        if enable_profiling:
            print("\nProfiling Summary:")
            print(f"{'=' * 50}")
            print("Top 10 time-consuming functions:")
            ps.print_stats(10)

        analyzed_data = analyze_data(stats.__dict__)
        return Result.ok(data=stats.__dict__, data_info=analyzed_data)
exit()

proxi attr

Source code in toolboxv2/utils/system/types.py
1897
1898
def exit(self):
    """proxi attr"""
exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1763
1764
1765
@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""
footprint(update_tracking=True)

Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

Parameters:

Name Type Description Default
update_tracking bool

Wenn True, aktualisiert Min/Max/Avg-Tracking

True

Returns:

Type Description
FootprintMetrics

FootprintMetrics mit allen erfassten Metriken

Source code in toolboxv2/utils/system/types.py
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
    """
    Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

    Args:
        update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

    Returns:
        FootprintMetrics mit allen erfassten Metriken
    """
    current_time = time.time()
    uptime_seconds = current_time - self._footprint_start_time

    # Formatierte Uptime
    uptime_delta = timedelta(seconds=int(uptime_seconds))
    uptime_formatted = str(uptime_delta)

    # Memory Metrics (in MB)
    try:
        mem_info = self._process.memory_info()
        memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
        memory_percent = self._process.memory_percent()

        if update_tracking:
            self._update_metric_tracking('memory', memory_current)

        memory_max = self._footprint_metrics['memory']['max']
        memory_min = self._footprint_metrics['memory']['min']
        if memory_min == float('inf'):
            memory_min = memory_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        memory_current = memory_max = memory_min = memory_percent = 0

    # CPU Metrics
    try:
        cpu_percent_current = self._process.cpu_percent(interval=0.1)
        cpu_times = self._process.cpu_times()
        cpu_time_seconds = cpu_times.user + cpu_times.system

        if update_tracking:
            self._update_metric_tracking('cpu', cpu_percent_current)

        cpu_percent_max = self._footprint_metrics['cpu']['max']
        cpu_percent_min = self._footprint_metrics['cpu']['min']
        cpu_percent_avg = self._get_metric_avg('cpu')

        if cpu_percent_min == float('inf'):
            cpu_percent_min = cpu_percent_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        cpu_percent_current = cpu_percent_max = 0
        cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

    # Disk I/O Metrics (in MB)
    try:
        io_counters = self._process.io_counters()
        disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
        disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

        disk_read_mb = disk_read_bytes / (1024 * 1024)
        disk_write_mb = disk_write_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('disk_read', disk_read_mb)
            self._update_metric_tracking('disk_write', disk_write_mb)

        disk_read_max = self._footprint_metrics['disk_read']['max']
        disk_read_min = self._footprint_metrics['disk_read']['min']
        disk_write_max = self._footprint_metrics['disk_write']['max']
        disk_write_min = self._footprint_metrics['disk_write']['min']

        if disk_read_min == float('inf'):
            disk_read_min = disk_read_mb
        if disk_write_min == float('inf'):
            disk_write_min = disk_write_mb
    except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
        disk_read_mb = disk_write_mb = 0
        disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

    # Network I/O Metrics (in MB)
    try:
        net_io = psutil.net_io_counters()
        network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
        network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

        network_sent_mb = network_sent_bytes / (1024 * 1024)
        network_recv_mb = network_recv_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('network_sent', network_sent_mb)
            self._update_metric_tracking('network_recv', network_recv_mb)

        network_sent_max = self._footprint_metrics['network_sent']['max']
        network_sent_min = self._footprint_metrics['network_sent']['min']
        network_recv_max = self._footprint_metrics['network_recv']['max']
        network_recv_min = self._footprint_metrics['network_recv']['min']

        if network_sent_min == float('inf'):
            network_sent_min = network_sent_mb
        if network_recv_min == float('inf'):
            network_recv_min = network_recv_mb
    except (AttributeError, OSError):
        network_sent_mb = network_recv_mb = 0
        network_sent_max = network_sent_min = 0
        network_recv_max = network_recv_min = 0

    # Process Info
    try:
        process_id = self._process.pid
        threads = self._process.num_threads()
        open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
        connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

        open_files = len(open_files_path)
        connections = len(connections_uri)
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        process_id = os.getpid()
        threads = open_files = connections = 0
        open_files_path = []
        connections_uri = []

    return FootprintMetrics(
        start_time=self._footprint_start_time,
        uptime_seconds=uptime_seconds,
        uptime_formatted=uptime_formatted,
        memory_current=memory_current,
        memory_max=memory_max,
        memory_min=memory_min,
        memory_percent=memory_percent,
        cpu_percent_current=cpu_percent_current,
        cpu_percent_max=cpu_percent_max,
        cpu_percent_min=cpu_percent_min,
        cpu_percent_avg=cpu_percent_avg,
        cpu_time_seconds=cpu_time_seconds,
        disk_read_mb=disk_read_mb,
        disk_write_mb=disk_write_mb,
        disk_read_max=disk_read_max,
        disk_read_min=disk_read_min,
        disk_write_max=disk_write_max,
        disk_write_min=disk_write_min,
        network_sent_mb=network_sent_mb,
        network_recv_mb=network_recv_mb,
        network_sent_max=network_sent_max,
        network_sent_min=network_sent_min,
        network_recv_max=network_recv_max,
        network_recv_min=network_recv_min,
        process_id=process_id,
        threads=threads,
        open_files=open_files,
        connections=connections,
        open_files_path=open_files_path,
        connections_uri=connections_uri,
    )
fuction_runner(function, function_data, args, kwargs, t0=0.0)

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py
1958
1959
1960
1961
1962
1963
1964
1965
1966
def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """
get_all_mods(working_dir='mods', path_to='./runtime')

proxi attr

Source code in toolboxv2/utils/system/types.py
1868
1869
def get_all_mods(self, working_dir="mods", path_to="./runtime"):
    """proxi attr"""
get_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py
2174
2175
def get_autocompletion_dict(self):
    """proxi attr"""
get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/system/types.py
1909
1910
1911
1912
1913
1914
1915
1916
1917
def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """
get_mod(name, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py
1997
1998
def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
    """proxi attr"""
get_username(get_input=False, default='loot')

proxi attr

Source code in toolboxv2/utils/system/types.py
2177
2178
def get_username(self, get_input=False, default="loot") -> str:
    """proxi attr"""
hide_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1767
1768
1769
@staticmethod
async def hide_console(*args, **kwargs):
    """proxi attr"""
inplace_load_instance(mod_name, loc='toolboxv2.mods.', spec='app', save=True)

proxi attr

Source code in toolboxv2/utils/system/types.py
1834
1835
def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
    """proxi attr"""
load_all_mods_in_file(working_dir='mods') async

proxi attr

Source code in toolboxv2/utils/system/types.py
1865
1866
async def load_all_mods_in_file(self, working_dir="mods"):
    """proxi attr"""
load_external_mods() async

proxi attr

Source code in toolboxv2/utils/system/types.py
1862
1863
async def load_external_mods(self):
    """proxi attr"""
load_mod(mod_name, mlm='I', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1856
1857
def load_mod(self, mod_name: str, mlm='I', **kwargs):
    """proxi attr"""
mod_online(mod_name, installed=False)

proxi attr

Source code in toolboxv2/utils/system/types.py
1843
1844
def mod_online(self, mod_name, installed=False):
    """proxi attr"""
print(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
2000
2001
2002
@staticmethod
def print(text, *args, **kwargs):
    """proxi attr"""
print_footprint(detailed=True)

Gibt den Footprint formatiert aus.

Parameters:

Name Type Description Default
detailed bool

Wenn True, zeigt alle Details, sonst nur Zusammenfassung

True

Returns:

Type Description
str

Formatierter Footprint-String

Source code in toolboxv2/utils/system/types.py
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
def print_footprint(self, detailed: bool = True) -> str:
    """
    Gibt den Footprint formatiert aus.

    Args:
        detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

    Returns:
        Formatierter Footprint-String
    """
    metrics = self.footprint()

    output = [
        "=" * 70,
        f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
        "=" * 70,
        f"\n📊 UPTIME",
        f"  Runtime: {metrics.uptime_formatted}",
        f"  Seconds: {metrics.uptime_seconds:.2f}s",
        f"\n💾 MEMORY USAGE",
        f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
        f"  Maximum:  {metrics.memory_max:.2f} MB",
        f"  Minimum:  {metrics.memory_min:.2f} MB",
    ]

    if detailed:
        helper_ = '\n\t- '.join(metrics.open_files_path)
        helper__ = '\n\t- '.join(metrics.connections_uri)
        output.extend([
            f"\n⚙️  CPU USAGE",
            f"  Current:  {metrics.cpu_percent_current:.2f}%",
            f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
            f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
            f"  Average:  {metrics.cpu_percent_avg:.2f}%",
            f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
            f"\n💿 DISK I/O",
            f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
            f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
            f"\n🌐 NETWORK I/O",
            f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
            f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
            f"\n🔧 PROCESS INFO",
            f"  PID:         {metrics.process_id}",
            f"  Threads:     {metrics.threads}",
            f"\n📂 OPEN FILES",
            f"  Open Files:  {metrics.open_files}",
            f"  Open Files Path: \n\t- {helper_}",
            f"\n🔗 NETWORK CONNECTIONS",
            f"  Connections: {metrics.connections}",
            f"  Connections URI: \n\t- {helper__}",
        ])

    output.append("=" * 70)

    return "\n".join(output)
print_ok()

proxi attr

Source code in toolboxv2/utils/system/types.py
1881
1882
1883
def print_ok(self):
    """proxi attr"""
    self.logger.info("OK")
reload_mod(mod_name, spec='app', is_file=True, loc='toolboxv2.mods.')

proxi attr

Source code in toolboxv2/utils/system/types.py
1885
1886
def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
    """proxi attr"""
remove_mod(mod_name, spec='app', delete=True)

proxi attr

Source code in toolboxv2/utils/system/types.py
1891
1892
def remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""
rrun_flows(name, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1796
1797
def rrun_flows(self, name, **kwargs):
    """proxi attr"""
run_a_from_sync(function, *args)

run a async fuction

Source code in toolboxv2/utils/system/types.py
1919
1920
1921
1922
def run_a_from_sync(self, function, *args):
    """
    run a async fuction
    """
run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1984
1985
1986
1987
1988
def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
            get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
            kwargs_=None,
            *args, **kwargs):
    """proxi attr"""
run_bg_task(task)

run a async fuction

Source code in toolboxv2/utils/system/types.py
1934
1935
1936
1937
def run_bg_task(self, task):
    """
            run a async fuction
            """
run_bg_task_advanced(task, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1924
1925
1926
1927
def run_bg_task_advanced(self, task, *args, **kwargs):
    """
    proxi attr
    """
run_flows(name, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1793
1794
async def run_flows(self, name, **kwargs):
    """proxi attr"""
run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1938
1939
1940
1941
1942
1943
1944
1945
1946
def run_function(self, mod_function_name: Enum or tuple,
                 tb_run_function_with_state=True,
                 tb_run_with_specification='app',
                 args_=None,
                 kwargs_=None,
                 *args,
                 **kwargs) -> Result:

    """proxi attr"""
run_http(mod_function_name, function_name=None, method='GET', args_=None, kwargs_=None, *args, **kwargs) async

run a function remote via http / https

Source code in toolboxv2/utils/system/types.py
1978
1979
1980
1981
1982
async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                   args_=None,
                   kwargs_=None,
                   *args, **kwargs):
    """run a function remote via http / https"""
save_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py
2171
2172
def save_autocompletion_dict(self):
    """proxi attr"""
save_exit()

proxi attr

Source code in toolboxv2/utils/system/types.py
1853
1854
def save_exit(self):
    """proxi attr"""
save_initialized_module(tools_class, spec)

proxi attr

Source code in toolboxv2/utils/system/types.py
1840
1841
def save_initialized_module(self, tools_class, spec):
    """proxi attr"""
save_instance(instance, modular_id, spec='app', instance_type='file/application', tools_class=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1837
1838
def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
    """proxi attr"""
save_load(modname, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py
1906
1907
def save_load(self, modname, spec='app'):
    """proxi attr"""
save_registry_as_enums(directory, filename)

proxi attr

Source code in toolboxv2/utils/system/types.py
2180
2181
def save_registry_as_enums(self, directory: str, filename: str):
    """proxi attr"""
set_flows(r)

proxi attr

Source code in toolboxv2/utils/system/types.py
1790
1791
def set_flows(self, r):
    """proxi attr"""
set_logger(debug=False)

proxi attr

Source code in toolboxv2/utils/system/types.py
1779
1780
def set_logger(self, debug=False):
    """proxi attr"""
show_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1771
1772
1773
@staticmethod
async def show_console(*args, **kwargs):
    """proxi attr"""
sprint(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
2004
2005
2006
@staticmethod
def sprint(text, *args, **kwargs):
    """proxi attr"""
tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, row=False, request_as_kwarg=False, state=None, level=0, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name Type Description Default
name str

The name to register the function under. Defaults to the function's own name.

None
mod_name str

The name of the module the function belongs to.

''
helper str

A helper string providing additional information about the function.

''
version str or None

The version of the function or module.

None
test bool

Flag to indicate if the function is for testing purposes.

True
restrict_in_virtual_mode bool

Flag to restrict the function in virtual mode.

False
api bool

Flag to indicate if the function is part of an API.

False
initial bool

Flag to indicate if the function should be executed at initialization.

False
exit_f bool

Flag to indicate if the function should be executed at exit.

False
test_only bool

Flag to indicate if the function should only be used for testing.

False
memory_cache bool

Flag to enable memory caching for the function.

False
request_as_kwarg bool

Flag to get request if the fuction is calld from api.

False
file_cache bool

Flag to enable file caching for the function.

False
row bool

rather to auto wrap the result in Result type default False means no row data aka result type

False
state bool or None

Flag to indicate if the function maintains state.

None
level int

The level of the function, used for prioritization or categorization.

0
memory_cache_max_size int

Maximum size of the memory cache.

100
memory_cache_ttl int

Time-to-live for the memory cache entries.

300
samples list or dict or None

Samples or examples of function usage.

None
interface str

The interface type for the function.

None
pre_compute callable

A function to be called before the main function.

None
post_compute callable

A function to be called after the main function.

None
api_methods list[str]

default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

None

Returns:

Name Type Description
function

The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/system/types.py
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str or None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       row=False,
       request_as_kwarg: bool = False,
       state: bool or None = None,
       level: int = 0,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  row=row,
                                  request_as_kwarg=request_as_kwarg,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl)
wait_for_bg_tasks(timeout=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1929
1930
1931
1932
def wait_for_bg_tasks(self, timeout=None):
    """
    proxi attr
    """
watch_mod(mod_name, spec='app', loc='toolboxv2.mods.', use_thread=True, path_name=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1888
1889
def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
    """proxi attr"""
web_context()

returns the build index ( toolbox web component )

Source code in toolboxv2/utils/system/types.py
1900
1901
def web_context(self) -> str:
    """returns the build index ( toolbox web component )"""
MainTool
Source code in toolboxv2/utils/system/main_tool.py
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()
__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")
__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py
174
175
176
177
178
179
180
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self
get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py
167
168
169
def get_version(self) -> str:
    """"Returns the version"""
    return self.version
webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py
164
165
def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""
MainToolType
Source code in toolboxv2/utils/system/types.py
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
class MainToolType:
    toolID: str
    app: A
    interface: ToolBoxInterfaces
    spec: str

    version: str
    tools: dict  # legacy
    name: str
    logger: logging
    color: str
    todo: Callable
    _on_exit: Callable
    stuf: bool
    config: dict
    user: U | None
    description: str

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None) -> Result:
        """proxi attr"""

    def load(self):
        """proxi attr"""

    def print(self, message, end="\n", **kwargs):
        """proxi attr"""

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    async def get_user(self, username: str) -> Result:
        return self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)
load()

proxi attr

Source code in toolboxv2/utils/system/types.py
1309
1310
def load(self):
    """proxi attr"""
print(message, end='\n', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1312
1313
def print(self, message, end="\n", **kwargs):
    """proxi attr"""
return_result(error=ToolBoxError.none, exec_code=0, help_text='', data_info=None, data=None, data_to=None) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1300
1301
1302
1303
1304
1305
1306
1307
@staticmethod
def return_result(error: ToolBoxError = ToolBoxError.none,
                  exec_code: int = 0,
                  help_text: str = "",
                  data_info=None,
                  data=None,
                  data_to=None) -> Result:
    """proxi attr"""
webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/types.py
1321
1322
def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""
Result
Source code in toolboxv2/utils/system/types.py
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task
__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py
643
644
645
646
647
648
649
650
651
652
653
def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult
binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)
cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py
738
739
740
741
742
743
744
745
746
747
748
def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result
file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name Type Description Default
data

File data as bytes or base64 string

required
filename

Name of the file for download

required
content_type

MIME type of the file (auto-detected if None)

None
info

Response info text

'OK'
interface

Target interface

remote

Returns:

Type Description

Result object configured for file download

Source code in toolboxv2/utils/system/types.py
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)
get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py
750
751
752
def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type
is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py
754
755
756
def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None
json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py
970
971
972
973
974
975
976
977
978
979
980
981
982
983
@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)
redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)
sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name Type Description Default
stream_generator Any

A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.

required
info str

Optional help text for the Result.

'OK'
interface ToolBoxInterfaces

Optional ToolBoxInterface to target.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional cleanup function to run when the stream ends or is cancelled.

None
#http_headers

Optional dictionary of custom HTTP headers for the SSE response.

required

Returns:

Type Description

A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )
stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name Type Description Default
stream_generator Any

Any stream source (async generator, sync generator, iterable, or single item).

required
content_type str

Content-Type header (default: text/event-stream for SSE).

'text/event-stream'
headers dict | None

Additional HTTP headers for the response.

None
info str

Help text for the result.

'OK'
interface ToolBoxInterfaces

Interface to send data to.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional function for cleanup.

None

Returns:

Type Description

A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)
text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)
typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py
667
668
669
670
671
672
673
674
675
676
async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data
typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py
655
656
657
658
659
660
661
662
663
664
665
def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data
typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance
typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py
705
706
707
708
709
710
711
712
713
714
715
716
@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance
all_functions_enums

Automatic generated by ToolBox v = 0.1.22

main_tool
MainTool
Source code in toolboxv2/utils/system/main_tool.py
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
class MainTool:
    toolID: str = ""
    # app = None
    interface = None
    spec = "app"
    name = ""
    color = "Bold"
    stuf = False

    def __init__(self, *args, **kwargs):
        """
        Standard constructor used for arguments pass
        Do not override. Use __ainit__ instead
        """
        self.__storedargs = args, kwargs
        self.tools = kwargs.get("tool", {})
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
        if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
            self.on_exit =self.app.tb(
                mod_name=self.name,
                name=kwargs.get("on_exit").__name__,
                version=self.version if hasattr(self, 'version') else "0.0.0",
            )(kwargs.get("on_exit"))
        self.async_initialized = False
        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    pass
                else:
                    self.todo()
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")

    async def __ainit__(self, *args, **kwargs):
        self.version = kwargs.get("v", kwargs.get("version", "0.0.0"))
        self.tools = kwargs.get("tool", {})
        self.name = kwargs["name"]
        self.logger = kwargs.get("logs", get_logger())
        self.color = kwargs.get("color", "WHITE")
        self.todo = kwargs.get("load", kwargs.get("on_start"))
        if not hasattr(self, 'config'):
            self.config = {}
        self.user = None
        self.description = "A toolbox mod" if kwargs.get("description") is None else kwargs.get("description")
        if MainTool.interface is None:
            MainTool.interface = self.app.interface_type
        # Result.default(self.app.interface)

        if self.todo:
            try:
                if inspect.iscoroutinefunction(self.todo):
                    await self.todo()
                else:
                    pass
                await asyncio.sleep(0.1)
                get_logger().info(f"{self.name} on load suspended")
            except Exception as e:
                get_logger().error(f"Error loading mod {self.name} {e}")
                if self.app.debug:
                    import traceback
                    traceback.print_exc()
        else:
            get_logger().info(f"{self.name} no load require")
        self.app.print(f"TOOL : {self.spec}.{self.name} online")



    @property
    def app(self):
        return get_app(
            from_=f"{self.spec}.{self.name}|{self.toolID if self.toolID else '*' + MainTool.toolID} {self.interface if self.interface else MainTool.interface}")

    @app.setter
    def app(self, v):
        raise PermissionError(f"You cannot set the App Instance! {v=}")

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None):

        if data_to is None:
            data_to = MainTool.interface if MainTool.interface is not None else ToolBoxInterfaces.cli

        if data is None:
            data = {}

        if data_info is None:
            data_info = {}

        return Result(
            error,
            ToolBoxResult(data_info=data_info, data=data, data_to=data_to),
            ToolBoxInfo(exec_code=exec_code, help_text=help_text)
        )

    def print(self, message, end="\n", **kwargs):
        if self.stuf:
            return

        self.app.print(Style.style_dic[self.color] + self.name + Style.style_dic["END"] + ":", message, end=end,
                       **kwargs)

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    def get_version(self) -> str:
        """"Returns the version"""
        return self.version

    async def get_user(self, username: str) -> Result:
        return await self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)

    async def __initobj(self):
        """Crutch used for __await__ after spawning"""
        assert not self.async_initialized
        self.async_initialized = True
        # pass the parameters to __ainit__ that passed to __init__
        await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
        return self

    def __await__(self):
        return self.__initobj().__await__()
__init__(*args, **kwargs)

Standard constructor used for arguments pass Do not override. Use ainit instead

Source code in toolboxv2/utils/system/main_tool.py
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def __init__(self, *args, **kwargs):
    """
    Standard constructor used for arguments pass
    Do not override. Use __ainit__ instead
    """
    self.__storedargs = args, kwargs
    self.tools = kwargs.get("tool", {})
    self.logger = kwargs.get("logs", get_logger())
    self.color = kwargs.get("color", "WHITE")
    self.todo = kwargs.get("load", kwargs.get("on_start", lambda: None))
    if "on_exit" in kwargs and isinstance(kwargs.get("on_exit"), Callable):
        self.on_exit =self.app.tb(
            mod_name=self.name,
            name=kwargs.get("on_exit").__name__,
            version=self.version if hasattr(self, 'version') else "0.0.0",
        )(kwargs.get("on_exit"))
    self.async_initialized = False
    if self.todo:
        try:
            if inspect.iscoroutinefunction(self.todo):
                pass
            else:
                self.todo()
            get_logger().info(f"{self.name} on load suspended")
        except Exception as e:
            get_logger().error(f"Error loading mod {self.name} {e}")
            if self.app.debug:
                import traceback
                traceback.print_exc()
    else:
        get_logger().info(f"{self.name} no load require")
__initobj() async

Crutch used for await after spawning

Source code in toolboxv2/utils/system/main_tool.py
174
175
176
177
178
179
180
async def __initobj(self):
    """Crutch used for __await__ after spawning"""
    assert not self.async_initialized
    self.async_initialized = True
    # pass the parameters to __ainit__ that passed to __init__
    await self.__ainit__(*self.__storedargs[0], **self.__storedargs[1])
    return self
get_version()

"Returns the version

Source code in toolboxv2/utils/system/main_tool.py
167
168
169
def get_version(self) -> str:
    """"Returns the version"""
    return self.version
webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/main_tool.py
164
165
def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""
get_version_from_pyproject(pyproject_path='../pyproject.toml')

Reads the version from the pyproject.toml file.

Source code in toolboxv2/utils/system/main_tool.py
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def get_version_from_pyproject(pyproject_path='../pyproject.toml'):
    """Reads the version from the pyproject.toml file."""
    if not os.path.exists(pyproject_path) and pyproject_path=='../pyproject.toml':
        pyproject_path = 'pyproject.toml'
    if not os.path.exists(pyproject_path) and pyproject_path=='pyproject.toml':
        return "0.1.21"

    try:
        import toml
        # Load the pyproject.toml file
        with open(pyproject_path) as file:
            pyproject_data = toml.load(file)

        # Extract the version from the 'project' section
        version = pyproject_data.get('project', {}).get('version')

        if version is None:
            raise ValueError(f"Version not found in {pyproject_path}")

        return version
    except Exception as e:
        print(f"Error reading version: {e}")
        return "0.0.0"
session
Session
Source code in toolboxv2/utils/system/session.py
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
class Session(metaclass=Singleton):

    # user: LocalUser

    def __init__(self, username, base=None):
        self.username = username
        self._session: ClientSession | None = None
        self._event_loop = None  # Track which event loop the session belongs to
        self.valid = False
        if base is None:
            base = os.environ.get("TOOLBOXV2_REMOTE_BASE", "https://simplecore.app")
        if base is not None and base.endswith("/api/"):
            base = base.replace("api/", "")
        self.base = base
        self.base = base.rstrip('/')  # Ensure no trailing slash

    @property
    def session(self):
        self._ensure_session()
        return self._session

    def _ensure_session(self):
        """Ensure session is valid for current event loop"""
        try:
            current_loop = asyncio.get_running_loop()
        except RuntimeError:
            # No running loop
            if self._session is not None:
                # Close old session if it exists
                self._session = None
                self._event_loop = None
            return

        # Check if session exists and is for the current loop
        if self._session is None or self._event_loop != current_loop:
            # Close old session if it exists and is from a different loop
            if self._session is not None:
                try:
                    # Try to close old session, but don't fail if loop is closed
                    if not self._session.closed:
                        asyncio.create_task(self._session.close())
                except:
                    pass
            # Create new session for current loop
            self._session = ClientSession()
            self._event_loop = current_loop

    async def init_log_in_mk_link(self, mak_link):
        from urllib.parse import parse_qs, urlparse
        await asyncio.sleep(0.1)

        pub_key, prv_key = Code.generate_asymmetric_keys()
        Code.save_keys_to_files(pub_key, prv_key, get_app('sys.session').info_dir.replace(get_app('sys.session').id, ''))
        parsed_url = urlparse(mak_link)
        params = parse_qs(parsed_url.query)
        invitation = params.get('key', [None])[0]
        self.username = params.get('name', [get_app('sys.session').get_username()])[0]
        if not invitation:
            print('Invalid LoginKey')
            return False

        res = await get_app("Session.InitLogin").run_http("CloudM.AuthManager", "add_user_device", method="POST",
                                                          name=self.username, pub_key=pub_key, invitation=invitation,
                                                          web_data=False, as_base64=False)
        res = Result.result_from_dict(**res)
        if res.is_error():
            return res
        await asyncio.sleep(0.1)
        return await self.auth_with_prv_key()

    @staticmethod
    def get_prv_key():
        pub_key, prv_key = Code.load_keys_from_files(get_app('sys.session.get_prv_key').info_dir.replace(get_app('sys.session.get_prv_key').id, ''))
        return prv_key

    def if_key(self):
        return len(self.get_prv_key()) > 0

    async def auth_with_prv_key(self):
        prv_key = self.get_prv_key()
        if not prv_key:
            return False
        challenge = await get_app("Session.InitLogin").run_http('CloudM.AuthManager', 'get_to_sing_data', method="POST",
                                                                args_='username=' + self.username + '&personal_key=False')
        challenge = Result.result_from_dict(**await challenge)
        if challenge.is_error():
            return challenge.lazy_return(-1, data=challenge.error)

        await asyncio.sleep(0.1)
        claim_data = await get_app("Session.InitLogin").run_http('CloudM.AuthManager', 'validate_device',
                                                                 username=self.username,
                                                                 signature=Code.create_signature(
                                                                     challenge.get("challenge"), prv_key,
                                                                     salt_length=32),
                                                                 method="POST")
        claim_data = Result.result_from_dict(**claim_data)
        if claim_data.is_error():
            return claim_data.lazy_return(-1, data=claim_data.error)

        claim = claim_data.get("key")

        if claim is None:
            return claim_data
        await asyncio.sleep(0.1)
        with BlobFile(f"claim/{self.username}/jwt.c", key=Code.DK()(), mode="w") as blob:
            blob.clear()
            blob.write(claim.encode())
        await asyncio.sleep(0.1)
        # Do something with the data or perform further actions
        res = await self.login()
        return res

    def init(self):
        self._ensure_session()

    async def login(self, verbose=False):
        self._ensure_session()
        with BlobFile(f"claim/{self.username}/jwt.c", key=Code.DK()(), mode="r") as blob:
            claim = blob.read()
            if claim == b'Error decoding':
                blob.clear()
                claim = b''
        if not claim:
            res = await self.auth_with_prv_key()
            return res is True
        try:
            self._ensure_session()  # Ensure session is valid before using
            async with self.session.request("POST", url=f"{self.base}/validateSession", json={'Jwt_claim': claim.decode(),
                                                                                             'Username': self.username}) as response:
                # print(response.status, "status")
                if response.status == 200:
                    print("Successfully Connected 2 TBxN")
                    get_logger().info("LogIn successful")
                    self.valid = True
                    return True
                if response.status == 401 and self.if_key():
                    return await self.auth_with_prv_key()
                # print(response)
                get_logger().warning("LogIn failed")
                return False
        except ClientConnectorError as e:
            print(f"Server nicht erreichbar (DNS oder Verbindung): {e}") if verbose else None
            return False
        except socket.gaierror as e:
            print(f"DNS-Auflösung fehlgeschlagen: {e}") if verbose else None
            return False
        except ClientError as e:
            print(f"Allgemeiner Client-Fehler: {e}") if verbose else None
            return False
        except Exception as e:
            print("Connection error", self.username, e) if verbose else None
            return False

    async def download_file(self, url, dest_folder="mods_sto"):
        self._ensure_session()
        if not self.session:
            raise Exception("Session not initialized. Please login first.")
        # Sicherstellen, dass das Zielverzeichnis existiert
        os.makedirs(dest_folder, exist_ok=True)

        # Analyse der URL, um den Dateinamen zu extrahieren
        filename = url.split('/')[-1]

        # Bereinigen des Dateinamens von Sonderzeichen
        valid_chars = '-_.()abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
        filename = ''.join(char for char in filename if char in valid_chars)

        # Konstruieren des vollständigen Dateipfads
        file_path = os.path.join(dest_folder, filename)
        if isinstance(url, str):
            url = self.base + url
        try:
            async with self.session.get(url) as response:
                if response.status == 200:
                    with open(file_path, 'wb') as f:
                        while True:
                            chunk = await response.content.read(1024)
                            if not chunk:
                                break
                            f.write(chunk)
                    print(f'File downloaded: {file_path}')
                    return True
                else:
                    print(f'Failed to download file: {url}. Status code: {response.status}')
        except ClientConnectorError as e:
            print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
            return False
        except socket.gaierror as e:
            print(f"DNS-Auflösung fehlgeschlagen: {e}")
            return False
        except ClientError as e:
            print(f"Allgemeiner Client-Fehler: {e}")
            return False
        except Exception as e:
            print("Error:",e, self.username)
        return False

    async def logout(self) -> bool:
        self._ensure_session()
        if self.session and not self.session.closed:  # Sicherstellen, dass die Session offen ist
            try:
                # Der `post`-Aufruf gibt eine Coroutine zurück, die wir awaiten müssen,
                # um das Response-Objekt zu erhalten.
                response = await self.session.post(f'{self.base}/web/logoutS')

                # Wir verwenden `async with` für die Response, um sicherzustellen, dass sie richtig behandelt wird.
                async with response:
                    is_successful = response.status == 200

                # Die Session erst NACH der Anfrage schließen.
                await self.session.close()
                self.session = None
                self._event_loop = None

                return is_successful
            except (ClientConnectorError, socket.gaierror, ClientError) as e:
                # Zusammengefasste Fehlerbehandlung für Verbindungsfehler
                print(f"Fehler bei der Verbindung während des Logouts: {e}")
                if self.session:
                    await self.session.close()
                self.session = None
                self._event_loop = None
                return False
            except Exception as e:
                print(f"Allgemeiner Fehler während des Logouts: {e}, user: {self.username}")
                if self.session:
                    await self.session.close()
                self.session = None
                self._event_loop = None
        return False  # Gibt False zurück, wenn keine Session vorhanden war

    async def fetch(self, url: str, method: str = 'GET', data=None, **kwargs) -> bool | ClientResponse | Response:
        # Ensure we have a valid session for the current event loop
        self._ensure_session()

        if isinstance(url, str):
            url = self.base + url
        if self.session:
            try:
                if method.upper() == 'POST':
                    return await self.session.post(url, json=data, **kwargs)
                else:
                    return await self.session.get(url, **kwargs)
            except ClientConnectorError as e:
                print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
                return False
            except socket.gaierror as e:
                print(f"DNS-Auflösung fehlgeschlagen: {e}")
                return False
            except ClientError as e:
                print(f"Allgemeiner Client-Fehler: {e}")
                return False
            except Exception as e:
                print("Error session fetch:", e, self.username)
                import traceback
                print(traceback.format_exc())
                return requests.request(method, url, data=data)
        else:
            print(f"Could not find session using request on {url}")
            if method.upper() == 'POST':
                return requests.request(method, url, json=data)
            return requests.request(method, url, data=data)
            # raise Exception("Session not initialized. Please login first.")

    async def upload_file(self, file_path: str, upload_url: str):
        # Prüfe, ob die Datei existiert
        if not os.path.isfile(file_path):
            raise FileNotFoundError(f"Datei {file_path} nicht gefunden.")

        # Initialisiere die Session, falls sie nicht bereits gestartet ist
        self._ensure_session()
        upload_url = self.base + upload_url
        # headers = {'accept': '*/*',
        #            'Content-Type': 'multipart/form-data'}
        # file_dict = {"file": open(file_path, "rb")}
        # response = await self.session.post(upload_url, headers=headers, data=file_dict)
        with open(file_path, 'rb') as f:
            file_data = f.read()

        with MultipartWriter('form-data') as mpwriter:
            part = mpwriter.append(file_data)
            part.set_content_disposition('form-data', name='file', filename=os.path.basename(file_path))
            try:
                async with self.session.post(upload_url, data=mpwriter, timeout=20000) as response:

                    # Prüfe, ob der Upload erfolgreich war
                    if response.status == 200:
                        print(f"Datei {file_path} erfolgreich hochgeladen.")
                        return await response.json()
                    else:
                        print(f"Fehler beim Hochladen der Datei {file_path}. Status: {response.status}")
                        print(await response.text())
                        return None
            except ClientConnectorError as e:
                print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
                return False
            except socket.gaierror as e:
                print(f"DNS-Auflösung fehlgeschlagen: {e}")
                return False
            except ClientError as e:
                print(f"Allgemeiner Client-Fehler: {e}")
                return False
            except Exception as e:
                print(f"Error session Fehler beim Hochladen der Datei {file_path}:", e, self.username)



    async def cleanup(self):
        """Cleanup session resources - should be called before application shutdown"""
        try:
            if self._session is not None and not self._session.closed:
                await self._session.close()
        except ClientConnectorError as e:
            print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
        except socket.gaierror as e:
            print(f"DNS-Auflösung fehlgeschlagen: {e}")
        except ClientError as e:
            print(f"Allgemeiner Client-Fehler: {e}")
        except Exception as e:
            pass
        finally:
            self._session = None
            self._event_loop = None

    def exit(self):
        with BlobFile(f"claim/{self.username}/jwt.c", key=Code.DK()(), mode="w") as blob:
            blob.clear()
cleanup() async

Cleanup session resources - should be called before application shutdown

Source code in toolboxv2/utils/system/session.py
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
async def cleanup(self):
    """Cleanup session resources - should be called before application shutdown"""
    try:
        if self._session is not None and not self._session.closed:
            await self._session.close()
    except ClientConnectorError as e:
        print(f"Server nicht erreichbar (DNS oder Verbindung): {e}")
    except socket.gaierror as e:
        print(f"DNS-Auflösung fehlgeschlagen: {e}")
    except ClientError as e:
        print(f"Allgemeiner Client-Fehler: {e}")
    except Exception as e:
        pass
    finally:
        self._session = None
        self._event_loop = None
state_system

The Task of the State System is : 1 Kep trak of the current state of the ToolBox and its dependency's 2 tracks the shasum of all mod and runnabael 3 the version of all mod

The state : {"utils":{"file_name": {"version":##,"shasum"}} ,"mods":{"file_name": {"version":##,"shasum":##,"src-url":##}} ,"runnable":{"file_name": {"version":##,"shasum":##,"src-url":##}} ,"api":{"file_name": {"version":##,"shasum"}} ,"app":{"file_name": {"version":##,"shasum":##,"src-url":##}} }

trans form state from on to an other.

detect_os_and_arch()

Detect the current operating system and architecture.

Source code in toolboxv2/utils/system/state_system.py
298
299
300
301
302
def detect_os_and_arch():
    """Detect the current operating system and architecture."""
    current_os = platform.system().lower()  # e.g., 'windows', 'linux', 'darwin'
    machine = platform.machine().lower()  # e.g., 'x86_64', 'amd64'
    return current_os, machine
download_executable(url, file_name)

Attempt to download the executable from the provided URL.

Source code in toolboxv2/utils/system/state_system.py
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
def download_executable(url, file_name):
    """Attempt to download the executable from the provided URL."""
    try:
        import requests
    except ImportError:
        print("The 'requests' library is required. Please install it via pip install requests")
        sys.exit(1)

    print(f"Attempting to download executable from {url}...")
    try:
        response = requests.get(url, stream=True)
    except Exception as e:
        print(f"Download error: {e}")
        return None

    if response.status_code == 200:
        with open(file_name, "wb") as f:
            for chunk in response.iter_content(chunk_size=8192):
                if chunk:
                    f.write(chunk)
        # Make the file executable on non-Windows systems
        if platform.system().lower() != "windows":
            os.chmod(file_name, 0o755)
        return file_name
    else:
        print("Download failed. Status code:", response.status_code)
        return None
find_highest_zip_version(name_filter, app_version=None, root_dir='mods_sto', version_only=False)

Findet die höchste verfügbare ZIP-Version in einem Verzeichnis basierend auf einem Namensfilter.

Parameters:

Name Type Description Default
root_dir str

Wurzelverzeichnis für die Suche

'mods_sto'
name_filter str

Namensfilter für die ZIP-Dateien

required
app_version str

Aktuelle App-Version für Kompatibilitätsprüfung

None

Returns:

Name Type Description
str str

Pfad zur ZIP-Datei mit der höchsten Version oder None wenn keine gefunden

Source code in toolboxv2/utils/system/state_system.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
def find_highest_zip_version(name_filter: str, app_version: str = None, root_dir: str = "mods_sto", version_only=False) -> str:
    """
    Findet die höchste verfügbare ZIP-Version in einem Verzeichnis basierend auf einem Namensfilter.

    Args:
        root_dir (str): Wurzelverzeichnis für die Suche
        name_filter (str): Namensfilter für die ZIP-Dateien
        app_version (str, optional): Aktuelle App-Version für Kompatibilitätsprüfung

    Returns:
        str: Pfad zur ZIP-Datei mit der höchsten Version oder None wenn keine gefunden
    """

    from packaging import version

    # Kompiliere den Regex-Pattern für die Dateinamen
    pattern = fr"{name_filter}&v[0-9.]+§([0-9.]+)\.zip$"

    highest_version = None
    highest_version_file = None

    # Durchsuche das Verzeichnis
    root_path = Path(root_dir)
    for file_path in root_path.rglob("*.zip"):
        if "RST$"+name_filter not in str(file_path):
            continue
        match = re.search(pattern, str(file_path).split("RST$")[-1].strip())
        if match:
            zip_version = match.group(1)

            # Prüfe App-Version Kompatibilität falls angegeben
            if app_version:
                file_app_version = re.search(r"&v([0-9.]+)§", str(file_path)).group(1)
                if version.parse(file_app_version) > version.parse(app_version):
                    continue

            # Vergleiche Versionen
            current_version = version.parse(zip_version)
            if highest_version is None or current_version > highest_version:
                highest_version = current_version
                highest_version_file = str(file_path)
    if version_only:
        return str(highest_version)
    return highest_version_file
find_highest_zip_version_entry(name, target_app_version=None, filepath='tbState.yaml')

Findet den Eintrag mit der höchsten ZIP-Version für einen gegebenen Namen und eine optionale Ziel-App-Version in einer YAML-Datei.

:param name: Der Name des gesuchten Eintrags. :param target_app_version: Die Zielversion der App als String (optional). :param filepath: Der Pfad zur YAML-Datei. :return: Den Eintrag mit der höchsten ZIP-Version innerhalb der Ziel-App-Version oder None, falls nicht gefunden.

Source code in toolboxv2/utils/system/state_system.py
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
def find_highest_zip_version_entry(name, target_app_version=None, filepath='tbState.yaml'):
    """
    Findet den Eintrag mit der höchsten ZIP-Version für einen gegebenen Namen und eine optionale Ziel-App-Version in einer YAML-Datei.

    :param name: Der Name des gesuchten Eintrags.
    :param target_app_version: Die Zielversion der App als String (optional).
    :param filepath: Der Pfad zur YAML-Datei.
    :return: Den Eintrag mit der höchsten ZIP-Version innerhalb der Ziel-App-Version oder None, falls nicht gefunden.
    """
    import yaml

    from packaging import version

    highest_zip_ver = None
    highest_entry = {}

    with open(filepath) as file:
        data = yaml.safe_load(file)
        # print(data)
        app_ver_h = None
        for key, value in list(data.get('installable', {}).items())[::-1]:
            # Prüfe, ob der Name im Schlüssel enthalten ist

            if name in key:
                v = value['version']
                if len(v) == 1:
                    app_ver = v[0].split('v')[-1]
                    zip_ver = "0.0.0"
                else:
                    app_ver, zip_ver = v
                    app_ver = app_ver.split('v')[-1]
                app_ver = version.parse(app_ver)
                # Wenn eine Ziel-App-Version angegeben ist, vergleiche sie
                if target_app_version is None or app_ver == version.parse(target_app_version):
                    current_zip_ver = version.parse(zip_ver)
                    # print(current_zip_ver, highest_zip_ver)

                    if highest_zip_ver is None or current_zip_ver > highest_zip_ver:
                        highest_zip_ver = current_zip_ver
                        highest_entry = value

                    if app_ver_h is None or app_ver > app_ver_h:
                        app_ver_h = app_ver
                        highest_zip_ver = current_zip_ver
                        highest_entry = value
    return highest_entry
query_executable_url(current_os, machine)

Query a remote URL for a matching executable based on OS and architecture. The file name is built dynamically based on parameters.

Source code in toolboxv2/utils/system/state_system.py
305
306
307
308
309
310
311
312
313
314
315
316
317
def query_executable_url(current_os, machine):
    """
    Query a remote URL for a matching executable based on OS and architecture.
    The file name is built dynamically based on parameters.
    """
    base_url = "https://example.com/downloads"  # Replace with the actual URL
    # Windows executables have .exe extension
    if current_os == "windows":
        file_name = f"server_{current_os}_{machine}.exe"
    else:
        file_name = f"server_{current_os}_{machine}"
    full_url = f"{base_url}/{file_name}"
    return full_url, file_name
types
AppType
Source code in toolboxv2/utils/system/types.py
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
class AppType:
    prefix: str
    id: str
    globals: dict[str, Any] = {"root": dict, }
    locals: dict[str, Any] = {"user": {'app': "self"}, }

    local_test: bool = False
    start_dir: str
    data_dir: str
    config_dir: str
    info_dir: str
    is_server:bool = False

    logger: logging.Logger
    logging_filename: str

    api_allowed_mods_list: list[str] = []

    version: str
    loop: asyncio.AbstractEventLoop

    keys: dict[str, str] = {
        "MACRO": "macro~~~~:",
        "MACRO_C": "m_color~~:",
        "HELPER": "helper~~~:",
        "debug": "debug~~~~:",
        "id": "name-spa~:",
        "st-load": "mute~load:",
        "comm-his": "comm-his~:",
        "develop-mode": "dev~mode~:",
        "provider::": "provider::",
    }

    defaults: dict[str, (bool or dict or dict[str, dict[str, str]] or str or list[str] or list[list]) | None] = {
        "MACRO": list[str],
        "MACRO_C": dict,
        "HELPER": dict,
        "debug": str,
        "id": str,
        "st-load": False,
        "comm-his": list[list],
        "develop-mode": bool,
    }

    cluster_manager: ClusterManager
    root_blob_storage: BlobStorage
    config_fh: FileHandler
    _debug: bool
    flows: dict[str, Callable]
    dev_modi: bool
    functions: dict[str, Any]
    modules: dict[str, Any]

    interface_type: ToolBoxInterfaces
    REFIX: str

    alive: bool
    called_exit: tuple[bool, float]
    args_sto: AppArgs
    system_flag = None
    session = None
    appdata = None
    exit_tasks = []

    enable_profiling: bool = False
    sto = None

    websocket_handlers: dict[str, dict[str, Callable]] = {}
    _rust_ws_bridge: Any = None

    docs_reader: Callable | None = None
    docs_writer: Callable | None = None
    get_update_suggestions: Callable | None = None
    auto_update_docs: Callable | None = None
    source_code_lookup: Callable | None = None

    initial_docs_parse: Callable | None = None

    def __init__(self, prefix=None, args=None):
        self.args_sto = args
        self.prefix = prefix
        self._footprint_start_time = time.time()
        self._process = psutil.Process(os.getpid())

        # Tracking-Daten für Min/Max/Avg
        self._footprint_metrics = {
            'memory': {'max': 0, 'min': float('inf'), 'samples': []},
            'cpu': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_read': {'max': 0, 'min': float('inf'), 'samples': []},
            'disk_write': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_sent': {'max': 0, 'min': float('inf'), 'samples': []},
            'network_recv': {'max': 0, 'min': float('inf'), 'samples': []},
        }

        # Initial Disk/Network Counters
        try:
            io_counters = self._process.io_counters()
            self._initial_disk_read = io_counters.read_bytes
            self._initial_disk_write = io_counters.write_bytes
        except (AttributeError, OSError):
            self._initial_disk_read = 0
            self._initial_disk_write = 0

        try:
            net_io = psutil.net_io_counters()
            self._initial_network_sent = net_io.bytes_sent
            self._initial_network_recv = net_io.bytes_recv
        except (AttributeError, OSError):
            self._initial_network_sent = 0
            self._initial_network_recv = 0

    def _update_metric_tracking(self, metric_name: str, value: float):
        """Aktualisiert Min/Max/Avg für eine Metrik"""
        metrics = self._footprint_metrics[metric_name]
        metrics['max'] = max(metrics['max'], value)
        metrics['min'] = min(metrics['min'], value)
        metrics['samples'].append(value)

        # Begrenze die Anzahl der Samples (letzte 1000)
        if len(metrics['samples']) > 1000:
            metrics['samples'] = metrics['samples'][-1000:]

    def _get_metric_avg(self, metric_name: str) -> float:
        """Berechnet Durchschnitt einer Metrik"""
        samples = self._footprint_metrics[metric_name]['samples']
        return sum(samples) / len(samples) if samples else 0

    def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
        """
        Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

        Args:
            update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

        Returns:
            FootprintMetrics mit allen erfassten Metriken
        """
        current_time = time.time()
        uptime_seconds = current_time - self._footprint_start_time

        # Formatierte Uptime
        uptime_delta = timedelta(seconds=int(uptime_seconds))
        uptime_formatted = str(uptime_delta)

        # Memory Metrics (in MB)
        try:
            mem_info = self._process.memory_info()
            memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
            memory_percent = self._process.memory_percent()

            if update_tracking:
                self._update_metric_tracking('memory', memory_current)

            memory_max = self._footprint_metrics['memory']['max']
            memory_min = self._footprint_metrics['memory']['min']
            if memory_min == float('inf'):
                memory_min = memory_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            memory_current = memory_max = memory_min = memory_percent = 0

        # CPU Metrics
        try:
            cpu_percent_current = self._process.cpu_percent(interval=0.1)
            cpu_times = self._process.cpu_times()
            cpu_time_seconds = cpu_times.user + cpu_times.system

            if update_tracking:
                self._update_metric_tracking('cpu', cpu_percent_current)

            cpu_percent_max = self._footprint_metrics['cpu']['max']
            cpu_percent_min = self._footprint_metrics['cpu']['min']
            cpu_percent_avg = self._get_metric_avg('cpu')

            if cpu_percent_min == float('inf'):
                cpu_percent_min = cpu_percent_current
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            cpu_percent_current = cpu_percent_max = 0
            cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

        # Disk I/O Metrics (in MB)
        try:
            io_counters = self._process.io_counters()
            disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
            disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

            disk_read_mb = disk_read_bytes / (1024 * 1024)
            disk_write_mb = disk_write_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('disk_read', disk_read_mb)
                self._update_metric_tracking('disk_write', disk_write_mb)

            disk_read_max = self._footprint_metrics['disk_read']['max']
            disk_read_min = self._footprint_metrics['disk_read']['min']
            disk_write_max = self._footprint_metrics['disk_write']['max']
            disk_write_min = self._footprint_metrics['disk_write']['min']

            if disk_read_min == float('inf'):
                disk_read_min = disk_read_mb
            if disk_write_min == float('inf'):
                disk_write_min = disk_write_mb
        except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
            disk_read_mb = disk_write_mb = 0
            disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

        # Network I/O Metrics (in MB)
        try:
            net_io = psutil.net_io_counters()
            network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
            network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

            network_sent_mb = network_sent_bytes / (1024 * 1024)
            network_recv_mb = network_recv_bytes / (1024 * 1024)

            if update_tracking:
                self._update_metric_tracking('network_sent', network_sent_mb)
                self._update_metric_tracking('network_recv', network_recv_mb)

            network_sent_max = self._footprint_metrics['network_sent']['max']
            network_sent_min = self._footprint_metrics['network_sent']['min']
            network_recv_max = self._footprint_metrics['network_recv']['max']
            network_recv_min = self._footprint_metrics['network_recv']['min']

            if network_sent_min == float('inf'):
                network_sent_min = network_sent_mb
            if network_recv_min == float('inf'):
                network_recv_min = network_recv_mb
        except (AttributeError, OSError):
            network_sent_mb = network_recv_mb = 0
            network_sent_max = network_sent_min = 0
            network_recv_max = network_recv_min = 0

        # Process Info
        try:
            process_id = self._process.pid
            threads = self._process.num_threads()
            open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
            connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

            open_files = len(open_files_path)
            connections = len(connections_uri)
        except (psutil.NoSuchProcess, psutil.AccessDenied):
            process_id = os.getpid()
            threads = open_files = connections = 0
            open_files_path = []
            connections_uri = []

        return FootprintMetrics(
            start_time=self._footprint_start_time,
            uptime_seconds=uptime_seconds,
            uptime_formatted=uptime_formatted,
            memory_current=memory_current,
            memory_max=memory_max,
            memory_min=memory_min,
            memory_percent=memory_percent,
            cpu_percent_current=cpu_percent_current,
            cpu_percent_max=cpu_percent_max,
            cpu_percent_min=cpu_percent_min,
            cpu_percent_avg=cpu_percent_avg,
            cpu_time_seconds=cpu_time_seconds,
            disk_read_mb=disk_read_mb,
            disk_write_mb=disk_write_mb,
            disk_read_max=disk_read_max,
            disk_read_min=disk_read_min,
            disk_write_max=disk_write_max,
            disk_write_min=disk_write_min,
            network_sent_mb=network_sent_mb,
            network_recv_mb=network_recv_mb,
            network_sent_max=network_sent_max,
            network_sent_min=network_sent_min,
            network_recv_max=network_recv_max,
            network_recv_min=network_recv_min,
            process_id=process_id,
            threads=threads,
            open_files=open_files,
            connections=connections,
            open_files_path=open_files_path,
            connections_uri=connections_uri,
        )

    def print_footprint(self, detailed: bool = True) -> str:
        """
        Gibt den Footprint formatiert aus.

        Args:
            detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

        Returns:
            Formatierter Footprint-String
        """
        metrics = self.footprint()

        output = [
            "=" * 70,
            f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
            "=" * 70,
            f"\n📊 UPTIME",
            f"  Runtime: {metrics.uptime_formatted}",
            f"  Seconds: {metrics.uptime_seconds:.2f}s",
            f"\n💾 MEMORY USAGE",
            f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
            f"  Maximum:  {metrics.memory_max:.2f} MB",
            f"  Minimum:  {metrics.memory_min:.2f} MB",
        ]

        if detailed:
            helper_ = '\n\t- '.join(metrics.open_files_path)
            helper__ = '\n\t- '.join(metrics.connections_uri)
            output.extend([
                f"\n⚙️  CPU USAGE",
                f"  Current:  {metrics.cpu_percent_current:.2f}%",
                f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
                f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
                f"  Average:  {metrics.cpu_percent_avg:.2f}%",
                f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
                f"\n💿 DISK I/O",
                f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
                f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
                f"\n🌐 NETWORK I/O",
                f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
                f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
                f"\n🔧 PROCESS INFO",
                f"  PID:         {metrics.process_id}",
                f"  Threads:     {metrics.threads}",
                f"\n📂 OPEN FILES",
                f"  Open Files:  {metrics.open_files}",
                f"  Open Files Path: \n\t- {helper_}",
                f"\n🔗 NETWORK CONNECTIONS",
                f"  Connections: {metrics.connections}",
                f"  Connections URI: \n\t- {helper__}",
            ])

        output.append("=" * 70)

        return "\n".join(output)



    def start_server(self):
        from toolboxv2.utils.clis.api import manage_server
        if self.is_server:
            return
        manage_server("start")
        self.is_server = False

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    async def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        """proxi attr"""

    @property
    def debug(self):
        """proxi attr"""
        return self._debug

    def debug_rains(self, e):
        """proxi attr"""

    def set_flows(self, r):
        """proxi attr"""

    async def run_flows(self, name, **kwargs):
        """proxi attr"""

    def rrun_flows(self, name, **kwargs):
        """proxi attr"""

    def idle(self):
        import time
        self.print("idle")
        try:
            while self.alive:
                time.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("idle done")

    async def a_idle(self):
        self.print("a idle (running :"+("online)" if hasattr(self, 'daemon_app') else "offline)"))
        try:
            if hasattr(self, 'daemon_app'):
                await self.daemon_app.connect(self)
            else:
                while self.alive:
                    await asyncio.sleep(1)
        except KeyboardInterrupt:
            pass
        self.print("a idle done")

    @debug.setter
    def debug(self, value):
        """proxi attr"""

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):
        """proxi attr"""

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        """proxi attr"""

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        """proxi attr"""

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
        """proxi attr"""

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
        """proxi attr"""

    def save_initialized_module(self, tools_class, spec):
        """proxi attr"""

    def mod_online(self, mod_name, installed=False):
        """proxi attr"""

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0):
        """proxi attr"""

    def save_exit(self):
        """proxi attr"""

    def load_mod(self, mod_name: str, mlm='I', **kwargs):
        """proxi attr"""

    async def init_module(self, modular):
        return await self.load_mod(modular)

    async def load_external_mods(self):
        """proxi attr"""

    async def load_all_mods_in_file(self, working_dir="mods"):
        """proxi attr"""

    def get_all_mods(self, working_dir="mods", path_to="./runtime"):
        """proxi attr"""

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    def print_ok(self):
        """proxi attr"""
        self.logger.info("OK")

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        """proxi attr"""

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
        """proxi attr"""

    def remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        """proxi attr"""

    def exit(self):
        """proxi attr"""

    def web_context(self) -> str:
        """returns the build index ( toolbox web component )"""

    async def a_exit(self):
        """proxi attr"""

    def save_load(self, modname, spec='app'):
        """proxi attr"""

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """

    def run_a_from_sync(self, function, *args):
        """
        run a async fuction
        """

    def run_bg_task_advanced(self, task, *args, **kwargs):
        """
        proxi attr
        """

    def wait_for_bg_tasks(self, timeout=None):
        """
        proxi attr
        """

    def run_bg_task(self, task):
        """
                run a async fuction
                """
    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        """proxi attr"""

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        """proxi attr"""

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
        """
        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        mod_function_name = f"{modular_name}.{function_name}"

        proxi attr
        """

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                       args_=None,
                       kwargs_=None,
                       *args, **kwargs):
        """run a function remote via http / https"""

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):
        """proxi attr"""

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):
        """proxi attr"""

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        """proxi attr"""

    @staticmethod
    def print(text, *args, **kwargs):
        """proxi attr"""

    @staticmethod
    def sprint(text, *args, **kwargs):
        """proxi attr"""

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def _register_function(self, module_name, func_name, data):
        """proxi attr"""

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial=False,
                          exit_f=False,
                          test=True,
                          samples=None,
                          state=None,
                          pre_compute=None,
                          post_compute=None,
                          memory_cache=False,
                          file_cache=False,
                          row=False,
                          request_as_kwarg=False,
                          memory_cache_max_size=100,
                          memory_cache_ttl=300,
                          websocket_handler: str | None = None,):
        """proxi attr"""

        # data = {
        #     "type": type_,
        #     "module_name": module_name,
        #     "func_name": func_name,
        #     "level": level,
        #     "restrict_in_virtual_mode": restrict_in_virtual_mode,
        #     "func": func,
        #     "api": api,
        #     "helper": helper,
        #     "version": version,
        #     "initial": initial,
        #     "exit_f": exit_f,
        #     "__module__": func.__module__,
        #     "signature": sig,
        #     "params": params,
        #     "state": (
        #         False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
        #     "do_test": test,
        #     "samples": samples,
        #     "request_as_kwarg": request_as_kwarg,

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str or None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           row=False,
           request_as_kwarg: bool = False,
           state: bool or None = None,
           level: int = 0,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      row=row,
                                      request_as_kwarg=request_as_kwarg,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl)

    def print_functions(self, name=None):


        if not self.functions:
            return

        def helper(_functions):
            for func_name, data in _functions.items():
                if not isinstance(data, dict):
                    continue

                func_type = data.get('type', 'Unknown')
                func_level = 'r' if data['level'] == -1 else data['level']
                api_status = 'Api' if data.get('api', False) else 'Non-Api'

                print(f"  Function: {func_name}{data.get('signature', '()')}; "
                      f"Type: {func_type}, Level: {func_level}, {api_status}")

        if name is not None:
            functions = self.functions.get(name)
            if functions is not None:
                print(f"\nModule: {name}; Type: {functions.get('app_instance_type', 'Unknown')}")
                helper(functions)
                return
        for module, functions in self.functions.items():
            print(f"\nModule: {module}; Type: {functions.get('app_instance_type', 'Unknown')}")
            helper(functions)

    def save_autocompletion_dict(self):
        """proxi attr"""

    def get_autocompletion_dict(self):
        """proxi attr"""

    def get_username(self, get_input=False, default="loot") -> str:
        """proxi attr"""

    def save_registry_as_enums(self, directory: str, filename: str):
        """proxi attr"""

    async def execute_all_functions_(self, m_query='', f_query=''):
        print("Executing all functions")
        from ..extras import generate_test_cases
        all_data = {
            "modular_run": 0,
            "modular_fatal_error": 0,
            "errors": 0,
            "modular_sug": 0,
            "coverage": [],
            "total_coverage": {},
        }
        items = list(self.functions.items()).copy()
        for module_name, functions in items:
            infos = {
                "functions_run": 0,
                "functions_fatal_error": 0,
                "error": 0,
                "functions_sug": 0,
                'calls': {},
                'callse': {},
                "coverage": [0, 0],
            }
            all_data['modular_run'] += 1
            if not module_name.startswith(m_query):
                all_data['modular_sug'] += 1
                continue

            with Spinner(message=f"In {module_name}| "):
                f_items = list(functions.items()).copy()
                for function_name, function_data in f_items:
                    if not isinstance(function_data, dict):
                        continue
                    if not function_name.startswith(f_query):
                        continue
                    test: list = function_data.get('do_test')
                    # print(test, module_name, function_name, function_data)
                    infos["coverage"][0] += 1
                    if test is False:
                        continue

                    with Spinner(message=f"\t\t\t\t\t\tfuction {function_name}..."):
                        params: list = function_data.get('params')
                        sig: signature = function_data.get('signature')
                        state: bool = function_data.get('state')
                        samples: bool = function_data.get('samples')

                        test_kwargs_list = [{}]

                        if params is not None:
                            test_kwargs_list = samples if samples is not None else generate_test_cases(sig=sig)
                            # print(test_kwargs)
                            # print(test_kwargs[0])
                            # test_kwargs = test_kwargs_list[0]
                        # print(module_name, function_name, test_kwargs_list)
                        infos["coverage"][1] += 1
                        for test_kwargs in test_kwargs_list:
                            try:
                                # print(f"test Running {state=} |{module_name}.{function_name}")
                                result = await self.a_run_function((module_name, function_name),
                                                                   tb_run_function_with_state=state,
                                                                   **test_kwargs)
                                if not isinstance(result, Result):
                                    result = Result.ok(result)
                                if result.info.exec_code == 0:
                                    infos['calls'][function_name] = [test_kwargs, str(result)]
                                    infos['functions_sug'] += 1
                                else:
                                    infos['functions_sug'] += 1
                                    infos['error'] += 1
                                    infos['callse'][function_name] = [test_kwargs, str(result)]
                            except Exception as e:
                                infos['functions_fatal_error'] += 1
                                infos['callse'][function_name] = [test_kwargs, str(e)]
                            finally:
                                infos['functions_run'] += 1

                if infos['functions_run'] == infos['functions_sug']:
                    all_data['modular_sug'] += 1
                else:
                    all_data['modular_fatal_error'] += 1
                if infos['error'] > 0:
                    all_data['errors'] += infos['error']

                all_data[module_name] = infos
                if infos['coverage'][0] == 0:
                    c = 0
                else:
                    c = infos['coverage'][1] / infos['coverage'][0]
                all_data["coverage"].append(f"{module_name}:{c:.2f}\n")
        total_coverage = sum([float(t.split(":")[-1]) for t in all_data["coverage"]]) / len(all_data["coverage"])
        print(
            f"\n{all_data['modular_run']=}\n{all_data['modular_sug']=}\n{all_data['modular_fatal_error']=}\n{total_coverage=}")
        d = analyze_data(all_data)
        return Result.ok(data=all_data, data_info=d)

    async def execute_function_test(self, module_name: str, function_name: str,
                                    function_data: dict, test_kwargs: dict,
                                    profiler: cProfile.Profile) -> tuple[bool, str, dict, float]:
        start_time = time.time()
        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            try:
                result = await self.a_run_function(
                    (module_name, function_name),
                    tb_run_function_with_state=function_data.get('state'),
                    **test_kwargs
                )

                if not isinstance(result, Result):
                    result = Result.ok(result)

                success = result.info.exec_code == 0
                execution_time = time.time() - start_time
                return success, str(result), test_kwargs, execution_time
            except Exception as e:
                execution_time = time.time() - start_time
                return False, str(e), test_kwargs, execution_time

    async def process_function(self, module_name: str, function_name: str,
                               function_data: dict, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()
        info = ModuleInfo()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            if not isinstance(function_data, dict):
                return function_name, info

            test = function_data.get('do_test')
            info.coverage[0] += 1

            if test is False:
                return function_name, info

            params = function_data.get('params')
            sig = function_data.get('signature')
            samples = function_data.get('samples')

            test_kwargs_list = [{}] if params is None else (
                samples if samples is not None else generate_test_cases(sig=sig)
            )

            info.coverage[1] += 1

            # Create tasks for all test cases
            tasks = [
                self.execute_function_test(module_name, function_name, function_data, test_kwargs, profiler)
                for test_kwargs in test_kwargs_list
            ]

            # Execute all tests concurrently
            results = await asyncio.gather(*tasks)

            total_execution_time = 0
            for success, result_str, test_kwargs, execution_time in results:
                info.functions_run += 1
                total_execution_time += execution_time

                if success:
                    info.functions_sug += 1
                    info.calls[function_name] = [test_kwargs, result_str]
                else:
                    info.functions_sug += 1
                    info.error += 1
                    info.callse[function_name] = [test_kwargs, result_str]

            info.execution_time = time.time() - start_time
            return function_name, info

    async def process_module(self, module_name: str, functions: dict,
                             f_query: str, profiler: cProfile.Profile) -> tuple[str, ModuleInfo]:
        start_time = time.time()

        with profile_section(profiler, hasattr(self, 'enable_profiling') and self.enable_profiling):
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_function(module_name, fname, fdata, profiler)
                    for fname, fdata in functions.items()
                    if fname.startswith(f_query)
                ]

                if not tasks:
                    return module_name, ModuleInfo()

                results = await asyncio.gather(*tasks)

                # Combine results from all functions in the module
                combined_info = ModuleInfo()
                total_execution_time = 0

                for _, info in results:
                    combined_info.functions_run += info.functions_run
                    combined_info.functions_fatal_error += info.functions_fatal_error
                    combined_info.error += info.error
                    combined_info.functions_sug += info.functions_sug
                    combined_info.calls.update(info.calls)
                    combined_info.callse.update(info.callse)
                    combined_info.coverage[0] += info.coverage[0]
                    combined_info.coverage[1] += info.coverage[1]
                    total_execution_time += info.execution_time

                combined_info.execution_time = time.time() - start_time
                return module_name, combined_info

    async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
        """
        Execute all functions with parallel processing and optional profiling.

        Args:
            m_query (str): Module name query filter
            f_query (str): Function name query filter
            enable_profiling (bool): Enable detailed profiling information
        """
        print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

        start_time = time.time()
        stats = ExecutionStats()
        items = list(self.functions.items()).copy()

        # Set up profiling
        self.enable_profiling = enable_profiling
        profiler = cProfile.Profile()

        with profile_section(profiler, enable_profiling):
            # Filter modules based on query
            filtered_modules = [
                (mname, mfuncs) for mname, mfuncs in items
                if mname.startswith(m_query)
            ]

            stats.modular_run = len(filtered_modules)

            # Process all modules concurrently
            async with asyncio.Semaphore(mp.cpu_count()):
                tasks = [
                    self.process_module(mname, mfuncs, f_query, profiler)
                    for mname, mfuncs in filtered_modules
                ]

                results = await asyncio.gather(*tasks)

            # Combine results and calculate statistics
            for module_name, info in results:
                if info.functions_run == info.functions_sug:
                    stats.modular_sug += 1
                else:
                    stats.modular_fatal_error += 1

                stats.errors += info.error

                # Calculate coverage
                coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
                stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

                # Store module info
                stats.__dict__[module_name] = info

            # Calculate total coverage
            total_coverage = (
                sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
                if stats.coverage else 0
            )

            stats.total_execution_time = time.time() - start_time

            # Generate profiling stats if enabled
            if enable_profiling:
                s = io.StringIO()
                ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
                ps.print_stats()
                stats.profiling_data = {
                    'detailed_stats': s.getvalue(),
                    'total_time': stats.total_execution_time,
                    'function_count': stats.modular_run,
                    'successful_functions': stats.modular_sug
                }

            print(
                f"\n{stats.modular_run=}"
                f"\n{stats.modular_sug=}"
                f"\n{stats.modular_fatal_error=}"
                f"\n{total_coverage=}"
                f"\nTotal execution time: {stats.total_execution_time:.2f}s"
            )

            if enable_profiling:
                print("\nProfiling Summary:")
                print(f"{'=' * 50}")
                print("Top 10 time-consuming functions:")
                ps.print_stats(10)

            analyzed_data = analyze_data(stats.__dict__)
            return Result.ok(data=stats.__dict__, data_info=analyzed_data)
debug property writable

proxi attr

a_exit() async

proxi attr

Source code in toolboxv2/utils/system/types.py
1903
1904
async def a_exit(self):
    """proxi attr"""
a_fuction_runner(function, function_data, args, kwargs) async

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py
1968
1969
1970
1971
1972
1973
1974
1975
1976
async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """
a_remove_mod(mod_name, spec='app', delete=True) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1894
1895
async def a_remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""
a_run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1990
1991
1992
1993
1994
1995
async def a_run_any(self, mod_function_name: Enum or str or tuple,
                    backwords_compability_variabel_string_holder=None,
                    get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                    kwargs_=None,
                    *args, **kwargs):
    """proxi attr"""
a_run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1948
1949
1950
1951
1952
1953
1954
1955
1956
async def a_run_function(self, mod_function_name: Enum or tuple,
                         tb_run_function_with_state=True,
                         tb_run_with_specification='app',
                         args_=None,
                         kwargs_=None,
                         *args,
                         **kwargs) -> Result:

    """proxi attr"""
debug_rains(e)

proxi attr

Source code in toolboxv2/utils/system/types.py
1787
1788
def debug_rains(self, e):
    """proxi attr"""
disconnect(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1775
1776
1777
@staticmethod
async def disconnect(*args, **kwargs):
    """proxi attr"""
execute_all_functions(m_query='', f_query='', enable_profiling=True) async

Execute all functions with parallel processing and optional profiling.

Parameters:

Name Type Description Default
m_query str

Module name query filter

''
f_query str

Function name query filter

''
enable_profiling bool

Enable detailed profiling information

True
Source code in toolboxv2/utils/system/types.py
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
async def execute_all_functions(self, m_query='', f_query='', enable_profiling=True):
    """
    Execute all functions with parallel processing and optional profiling.

    Args:
        m_query (str): Module name query filter
        f_query (str): Function name query filter
        enable_profiling (bool): Enable detailed profiling information
    """
    print("Executing all functions in parallel" + (" with profiling" if enable_profiling else ""))

    start_time = time.time()
    stats = ExecutionStats()
    items = list(self.functions.items()).copy()

    # Set up profiling
    self.enable_profiling = enable_profiling
    profiler = cProfile.Profile()

    with profile_section(profiler, enable_profiling):
        # Filter modules based on query
        filtered_modules = [
            (mname, mfuncs) for mname, mfuncs in items
            if mname.startswith(m_query)
        ]

        stats.modular_run = len(filtered_modules)

        # Process all modules concurrently
        async with asyncio.Semaphore(mp.cpu_count()):
            tasks = [
                self.process_module(mname, mfuncs, f_query, profiler)
                for mname, mfuncs in filtered_modules
            ]

            results = await asyncio.gather(*tasks)

        # Combine results and calculate statistics
        for module_name, info in results:
            if info.functions_run == info.functions_sug:
                stats.modular_sug += 1
            else:
                stats.modular_fatal_error += 1

            stats.errors += info.error

            # Calculate coverage
            coverage = (info.coverage[1] / info.coverage[0]) if info.coverage[0] > 0 else 0
            stats.coverage.append(f"{module_name}:{coverage:.2f}\n")

            # Store module info
            stats.__dict__[module_name] = info

        # Calculate total coverage
        total_coverage = (
            sum(float(t.split(":")[-1]) for t in stats.coverage) / len(stats.coverage)
            if stats.coverage else 0
        )

        stats.total_execution_time = time.time() - start_time

        # Generate profiling stats if enabled
        if enable_profiling:
            s = io.StringIO()
            ps = pstats.Stats(profiler, stream=s).sort_stats('cumulative')
            ps.print_stats()
            stats.profiling_data = {
                'detailed_stats': s.getvalue(),
                'total_time': stats.total_execution_time,
                'function_count': stats.modular_run,
                'successful_functions': stats.modular_sug
            }

        print(
            f"\n{stats.modular_run=}"
            f"\n{stats.modular_sug=}"
            f"\n{stats.modular_fatal_error=}"
            f"\n{total_coverage=}"
            f"\nTotal execution time: {stats.total_execution_time:.2f}s"
        )

        if enable_profiling:
            print("\nProfiling Summary:")
            print(f"{'=' * 50}")
            print("Top 10 time-consuming functions:")
            ps.print_stats(10)

        analyzed_data = analyze_data(stats.__dict__)
        return Result.ok(data=stats.__dict__, data_info=analyzed_data)
exit()

proxi attr

Source code in toolboxv2/utils/system/types.py
1897
1898
def exit(self):
    """proxi attr"""
exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1763
1764
1765
@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""
footprint(update_tracking=True)

Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

Parameters:

Name Type Description Default
update_tracking bool

Wenn True, aktualisiert Min/Max/Avg-Tracking

True

Returns:

Type Description
FootprintMetrics

FootprintMetrics mit allen erfassten Metriken

Source code in toolboxv2/utils/system/types.py
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
def footprint(self, update_tracking: bool = True) -> FootprintMetrics:
    """
    Erfasst den aktuellen Ressourcen-Footprint der Toolbox-Instanz.

    Args:
        update_tracking: Wenn True, aktualisiert Min/Max/Avg-Tracking

    Returns:
        FootprintMetrics mit allen erfassten Metriken
    """
    current_time = time.time()
    uptime_seconds = current_time - self._footprint_start_time

    # Formatierte Uptime
    uptime_delta = timedelta(seconds=int(uptime_seconds))
    uptime_formatted = str(uptime_delta)

    # Memory Metrics (in MB)
    try:
        mem_info = self._process.memory_info()
        memory_current = mem_info.rss / (1024 * 1024)  # Bytes zu MB
        memory_percent = self._process.memory_percent()

        if update_tracking:
            self._update_metric_tracking('memory', memory_current)

        memory_max = self._footprint_metrics['memory']['max']
        memory_min = self._footprint_metrics['memory']['min']
        if memory_min == float('inf'):
            memory_min = memory_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        memory_current = memory_max = memory_min = memory_percent = 0

    # CPU Metrics
    try:
        cpu_percent_current = self._process.cpu_percent(interval=0.1)
        cpu_times = self._process.cpu_times()
        cpu_time_seconds = cpu_times.user + cpu_times.system

        if update_tracking:
            self._update_metric_tracking('cpu', cpu_percent_current)

        cpu_percent_max = self._footprint_metrics['cpu']['max']
        cpu_percent_min = self._footprint_metrics['cpu']['min']
        cpu_percent_avg = self._get_metric_avg('cpu')

        if cpu_percent_min == float('inf'):
            cpu_percent_min = cpu_percent_current
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        cpu_percent_current = cpu_percent_max = 0
        cpu_percent_min = cpu_percent_avg = cpu_time_seconds = 0

    # Disk I/O Metrics (in MB)
    try:
        io_counters = self._process.io_counters()
        disk_read_bytes = io_counters.read_bytes - self._initial_disk_read
        disk_write_bytes = io_counters.write_bytes - self._initial_disk_write

        disk_read_mb = disk_read_bytes / (1024 * 1024)
        disk_write_mb = disk_write_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('disk_read', disk_read_mb)
            self._update_metric_tracking('disk_write', disk_write_mb)

        disk_read_max = self._footprint_metrics['disk_read']['max']
        disk_read_min = self._footprint_metrics['disk_read']['min']
        disk_write_max = self._footprint_metrics['disk_write']['max']
        disk_write_min = self._footprint_metrics['disk_write']['min']

        if disk_read_min == float('inf'):
            disk_read_min = disk_read_mb
        if disk_write_min == float('inf'):
            disk_write_min = disk_write_mb
    except (AttributeError, OSError, psutil.NoSuchProcess, psutil.AccessDenied):
        disk_read_mb = disk_write_mb = 0
        disk_read_max = disk_read_min = disk_write_max = disk_write_min = 0

    # Network I/O Metrics (in MB)
    try:
        net_io = psutil.net_io_counters()
        network_sent_bytes = net_io.bytes_sent - self._initial_network_sent
        network_recv_bytes = net_io.bytes_recv - self._initial_network_recv

        network_sent_mb = network_sent_bytes / (1024 * 1024)
        network_recv_mb = network_recv_bytes / (1024 * 1024)

        if update_tracking:
            self._update_metric_tracking('network_sent', network_sent_mb)
            self._update_metric_tracking('network_recv', network_recv_mb)

        network_sent_max = self._footprint_metrics['network_sent']['max']
        network_sent_min = self._footprint_metrics['network_sent']['min']
        network_recv_max = self._footprint_metrics['network_recv']['max']
        network_recv_min = self._footprint_metrics['network_recv']['min']

        if network_sent_min == float('inf'):
            network_sent_min = network_sent_mb
        if network_recv_min == float('inf'):
            network_recv_min = network_recv_mb
    except (AttributeError, OSError):
        network_sent_mb = network_recv_mb = 0
        network_sent_max = network_sent_min = 0
        network_recv_max = network_recv_min = 0

    # Process Info
    try:
        process_id = self._process.pid
        threads = self._process.num_threads()
        open_files_path = [str(x.path).replace("\\", "/") for x in self._process.open_files()]
        connections_uri = [f"{x.laddr}:{x.raddr} {str(x.status)}" for x in self._process.connections()]

        open_files = len(open_files_path)
        connections = len(connections_uri)
    except (psutil.NoSuchProcess, psutil.AccessDenied):
        process_id = os.getpid()
        threads = open_files = connections = 0
        open_files_path = []
        connections_uri = []

    return FootprintMetrics(
        start_time=self._footprint_start_time,
        uptime_seconds=uptime_seconds,
        uptime_formatted=uptime_formatted,
        memory_current=memory_current,
        memory_max=memory_max,
        memory_min=memory_min,
        memory_percent=memory_percent,
        cpu_percent_current=cpu_percent_current,
        cpu_percent_max=cpu_percent_max,
        cpu_percent_min=cpu_percent_min,
        cpu_percent_avg=cpu_percent_avg,
        cpu_time_seconds=cpu_time_seconds,
        disk_read_mb=disk_read_mb,
        disk_write_mb=disk_write_mb,
        disk_read_max=disk_read_max,
        disk_read_min=disk_read_min,
        disk_write_max=disk_write_max,
        disk_write_min=disk_write_min,
        network_sent_mb=network_sent_mb,
        network_recv_mb=network_recv_mb,
        network_sent_max=network_sent_max,
        network_sent_min=network_sent_min,
        network_recv_max=network_recv_max,
        network_recv_min=network_recv_min,
        process_id=process_id,
        threads=threads,
        open_files=open_files,
        connections=connections,
        open_files_path=open_files_path,
        connections_uri=connections_uri,
    )
fuction_runner(function, function_data, args, kwargs, t0=0.0)

parameters = function_data.get('params') modular_name = function_data.get('module_name') function_name = function_data.get('func_name') mod_function_name = f"{modular_name}.{function_name}"

proxi attr

Source code in toolboxv2/utils/system/types.py
1958
1959
1960
1961
1962
1963
1964
1965
1966
def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):
    """
    parameters = function_data.get('params')
    modular_name = function_data.get('module_name')
    function_name = function_data.get('func_name')
    mod_function_name = f"{modular_name}.{function_name}"

    proxi attr
    """
get_all_mods(working_dir='mods', path_to='./runtime')

proxi attr

Source code in toolboxv2/utils/system/types.py
1868
1869
def get_all_mods(self, working_dir="mods", path_to="./runtime"):
    """proxi attr"""
get_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py
2174
2175
def get_autocompletion_dict(self):
    """proxi attr"""
get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/system/types.py
1909
1910
1911
1912
1913
1914
1915
1916
1917
def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """
get_mod(name, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py
1997
1998
def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
    """proxi attr"""
get_username(get_input=False, default='loot')

proxi attr

Source code in toolboxv2/utils/system/types.py
2177
2178
def get_username(self, get_input=False, default="loot") -> str:
    """proxi attr"""
hide_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1767
1768
1769
@staticmethod
async def hide_console(*args, **kwargs):
    """proxi attr"""
inplace_load_instance(mod_name, loc='toolboxv2.mods.', spec='app', save=True)

proxi attr

Source code in toolboxv2/utils/system/types.py
1834
1835
def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True):
    """proxi attr"""
load_all_mods_in_file(working_dir='mods') async

proxi attr

Source code in toolboxv2/utils/system/types.py
1865
1866
async def load_all_mods_in_file(self, working_dir="mods"):
    """proxi attr"""
load_external_mods() async

proxi attr

Source code in toolboxv2/utils/system/types.py
1862
1863
async def load_external_mods(self):
    """proxi attr"""
load_mod(mod_name, mlm='I', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1856
1857
def load_mod(self, mod_name: str, mlm='I', **kwargs):
    """proxi attr"""
mod_online(mod_name, installed=False)

proxi attr

Source code in toolboxv2/utils/system/types.py
1843
1844
def mod_online(self, mod_name, installed=False):
    """proxi attr"""
print(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
2000
2001
2002
@staticmethod
def print(text, *args, **kwargs):
    """proxi attr"""
print_footprint(detailed=True)

Gibt den Footprint formatiert aus.

Parameters:

Name Type Description Default
detailed bool

Wenn True, zeigt alle Details, sonst nur Zusammenfassung

True

Returns:

Type Description
str

Formatierter Footprint-String

Source code in toolboxv2/utils/system/types.py
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
def print_footprint(self, detailed: bool = True) -> str:
    """
    Gibt den Footprint formatiert aus.

    Args:
        detailed: Wenn True, zeigt alle Details, sonst nur Zusammenfassung

    Returns:
        Formatierter Footprint-String
    """
    metrics = self.footprint()

    output = [
        "=" * 70,
        f"TOOLBOX FOOTPRINT - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}",
        "=" * 70,
        f"\n📊 UPTIME",
        f"  Runtime: {metrics.uptime_formatted}",
        f"  Seconds: {metrics.uptime_seconds:.2f}s",
        f"\n💾 MEMORY USAGE",
        f"  Current:  {metrics.memory_current:.2f} MB ({metrics.memory_percent:.2f}%)",
        f"  Maximum:  {metrics.memory_max:.2f} MB",
        f"  Minimum:  {metrics.memory_min:.2f} MB",
    ]

    if detailed:
        helper_ = '\n\t- '.join(metrics.open_files_path)
        helper__ = '\n\t- '.join(metrics.connections_uri)
        output.extend([
            f"\n⚙️  CPU USAGE",
            f"  Current:  {metrics.cpu_percent_current:.2f}%",
            f"  Maximum:  {metrics.cpu_percent_max:.2f}%",
            f"  Minimum:  {metrics.cpu_percent_min:.2f}%",
            f"  Average:  {metrics.cpu_percent_avg:.2f}%",
            f"  CPU Time: {metrics.cpu_time_seconds:.2f}s",
            f"\n💿 DISK I/O",
            f"  Read:     {metrics.disk_read_mb:.2f} MB (Max: {metrics.disk_read_max:.2f}, Min: {metrics.disk_read_min:.2f})",
            f"  Write:    {metrics.disk_write_mb:.2f} MB (Max: {metrics.disk_write_max:.2f}, Min: {metrics.disk_write_min:.2f})",
            f"\n🌐 NETWORK I/O",
            f"  Sent:     {metrics.network_sent_mb:.2f} MB (Max: {metrics.network_sent_max:.2f}, Min: {metrics.network_sent_min:.2f})",
            f"  Received: {metrics.network_recv_mb:.2f} MB (Max: {metrics.network_recv_max:.2f}, Min: {metrics.network_recv_min:.2f})",
            f"\n🔧 PROCESS INFO",
            f"  PID:         {metrics.process_id}",
            f"  Threads:     {metrics.threads}",
            f"\n📂 OPEN FILES",
            f"  Open Files:  {metrics.open_files}",
            f"  Open Files Path: \n\t- {helper_}",
            f"\n🔗 NETWORK CONNECTIONS",
            f"  Connections: {metrics.connections}",
            f"  Connections URI: \n\t- {helper__}",
        ])

    output.append("=" * 70)

    return "\n".join(output)
print_ok()

proxi attr

Source code in toolboxv2/utils/system/types.py
1881
1882
1883
def print_ok(self):
    """proxi attr"""
    self.logger.info("OK")
reload_mod(mod_name, spec='app', is_file=True, loc='toolboxv2.mods.')

proxi attr

Source code in toolboxv2/utils/system/types.py
1885
1886
def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
    """proxi attr"""
remove_mod(mod_name, spec='app', delete=True)

proxi attr

Source code in toolboxv2/utils/system/types.py
1891
1892
def remove_mod(self, mod_name, spec='app', delete=True):
    """proxi attr"""
rrun_flows(name, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1796
1797
def rrun_flows(self, name, **kwargs):
    """proxi attr"""
run_a_from_sync(function, *args)

run a async fuction

Source code in toolboxv2/utils/system/types.py
1919
1920
1921
1922
def run_a_from_sync(self, function, *args):
    """
    run a async fuction
    """
run_any(mod_function_name, backwords_compability_variabel_string_holder=None, get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1984
1985
1986
1987
1988
def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
            get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
            kwargs_=None,
            *args, **kwargs):
    """proxi attr"""
run_bg_task(task)

run a async fuction

Source code in toolboxv2/utils/system/types.py
1934
1935
1936
1937
def run_bg_task(self, task):
    """
            run a async fuction
            """
run_bg_task_advanced(task, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1924
1925
1926
1927
def run_bg_task_advanced(self, task, *args, **kwargs):
    """
    proxi attr
    """
run_flows(name, **kwargs) async

proxi attr

Source code in toolboxv2/utils/system/types.py
1793
1794
async def run_flows(self, name, **kwargs):
    """proxi attr"""
run_function(mod_function_name, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None, kwargs_=None, *args, **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1938
1939
1940
1941
1942
1943
1944
1945
1946
def run_function(self, mod_function_name: Enum or tuple,
                 tb_run_function_with_state=True,
                 tb_run_with_specification='app',
                 args_=None,
                 kwargs_=None,
                 *args,
                 **kwargs) -> Result:

    """proxi attr"""
run_http(mod_function_name, function_name=None, method='GET', args_=None, kwargs_=None, *args, **kwargs) async

run a function remote via http / https

Source code in toolboxv2/utils/system/types.py
1978
1979
1980
1981
1982
async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None, method="GET",
                   args_=None,
                   kwargs_=None,
                   *args, **kwargs):
    """run a function remote via http / https"""
save_autocompletion_dict()

proxi attr

Source code in toolboxv2/utils/system/types.py
2171
2172
def save_autocompletion_dict(self):
    """proxi attr"""
save_exit()

proxi attr

Source code in toolboxv2/utils/system/types.py
1853
1854
def save_exit(self):
    """proxi attr"""
save_initialized_module(tools_class, spec)

proxi attr

Source code in toolboxv2/utils/system/types.py
1840
1841
def save_initialized_module(self, tools_class, spec):
    """proxi attr"""
save_instance(instance, modular_id, spec='app', instance_type='file/application', tools_class=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1837
1838
def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):
    """proxi attr"""
save_load(modname, spec='app')

proxi attr

Source code in toolboxv2/utils/system/types.py
1906
1907
def save_load(self, modname, spec='app'):
    """proxi attr"""
save_registry_as_enums(directory, filename)

proxi attr

Source code in toolboxv2/utils/system/types.py
2180
2181
def save_registry_as_enums(self, directory: str, filename: str):
    """proxi attr"""
set_flows(r)

proxi attr

Source code in toolboxv2/utils/system/types.py
1790
1791
def set_flows(self, r):
    """proxi attr"""
set_logger(debug=False)

proxi attr

Source code in toolboxv2/utils/system/types.py
1779
1780
def set_logger(self, debug=False):
    """proxi attr"""
show_console(*args, **kwargs) async staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1771
1772
1773
@staticmethod
async def show_console(*args, **kwargs):
    """proxi attr"""
sprint(text, *args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
2004
2005
2006
@staticmethod
def sprint(text, *args, **kwargs):
    """proxi attr"""
tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, row=False, request_as_kwarg=False, state=None, level=0, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name Type Description Default
name str

The name to register the function under. Defaults to the function's own name.

None
mod_name str

The name of the module the function belongs to.

''
helper str

A helper string providing additional information about the function.

''
version str or None

The version of the function or module.

None
test bool

Flag to indicate if the function is for testing purposes.

True
restrict_in_virtual_mode bool

Flag to restrict the function in virtual mode.

False
api bool

Flag to indicate if the function is part of an API.

False
initial bool

Flag to indicate if the function should be executed at initialization.

False
exit_f bool

Flag to indicate if the function should be executed at exit.

False
test_only bool

Flag to indicate if the function should only be used for testing.

False
memory_cache bool

Flag to enable memory caching for the function.

False
request_as_kwarg bool

Flag to get request if the fuction is calld from api.

False
file_cache bool

Flag to enable file caching for the function.

False
row bool

rather to auto wrap the result in Result type default False means no row data aka result type

False
state bool or None

Flag to indicate if the function maintains state.

None
level int

The level of the function, used for prioritization or categorization.

0
memory_cache_max_size int

Maximum size of the memory cache.

100
memory_cache_ttl int

Time-to-live for the memory cache entries.

300
samples list or dict or None

Samples or examples of function usage.

None
interface str

The interface type for the function.

None
pre_compute callable

A function to be called before the main function.

None
post_compute callable

A function to be called after the main function.

None
api_methods list[str]

default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

None

Returns:

Name Type Description
function

The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/system/types.py
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str or None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       row=False,
       request_as_kwarg: bool = False,
       state: bool or None = None,
       level: int = 0,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  row=row,
                                  request_as_kwarg=request_as_kwarg,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl)
wait_for_bg_tasks(timeout=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1929
1930
1931
1932
def wait_for_bg_tasks(self, timeout=None):
    """
    proxi attr
    """
watch_mod(mod_name, spec='app', loc='toolboxv2.mods.', use_thread=True, path_name=None)

proxi attr

Source code in toolboxv2/utils/system/types.py
1888
1889
def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None):
    """proxi attr"""
web_context()

returns the build index ( toolbox web component )

Source code in toolboxv2/utils/system/types.py
1900
1901
def web_context(self) -> str:
    """returns the build index ( toolbox web component )"""
FootprintMetrics

Dataclass für Footprint-Metriken

Source code in toolboxv2/utils/system/types.py
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
@dataclass
class FootprintMetrics:
    """Dataclass für Footprint-Metriken"""
    # Uptime
    start_time: float
    uptime_seconds: float
    uptime_formatted: str

    # Memory (in MB)
    memory_current: float
    memory_max: float
    memory_min: float
    memory_percent: float

    # CPU
    cpu_percent_current: float
    cpu_percent_max: float
    cpu_percent_min: float
    cpu_percent_avg: float
    cpu_time_seconds: float

    # Disk I/O (in MB)
    disk_read_mb: float
    disk_write_mb: float
    disk_read_max: float
    disk_read_min: float
    disk_write_max: float
    disk_write_min: float

    # Network I/O (in MB)
    network_sent_mb: float
    network_recv_mb: float
    network_sent_max: float
    network_sent_min: float
    network_recv_max: float
    network_recv_min: float

    # Additional Info
    process_id: int
    threads: int
    open_files: int
    connections: int

    open_files_path: list[str]
    connections_uri: list[str]

    def to_dict(self) -> Dict[str, Any]:
        """Konvertiert Metriken in Dictionary"""
        return {
            'uptime': {
                'seconds': self.uptime_seconds,
                'formatted': self.uptime_formatted,
            },
            'memory': {
                'current_mb': self.memory_current,
                'max_mb': self.memory_max,
                'min_mb': self.memory_min,
                'percent': self.memory_percent,
            },
            'cpu': {
                'current_percent': self.cpu_percent_current,
                'max_percent': self.cpu_percent_max,
                'min_percent': self.cpu_percent_min,
                'avg_percent': self.cpu_percent_avg,
                'time_seconds': self.cpu_time_seconds,
            },
            'disk': {
                'read_mb': self.disk_read_mb,
                'write_mb': self.disk_write_mb,
                'read_max_mb': self.disk_read_max,
                'read_min_mb': self.disk_read_min,
                'write_max_mb': self.disk_write_max,
                'write_min_mb': self.disk_write_min,
            },
            'network': {
                'sent_mb': self.network_sent_mb,
                'recv_mb': self.network_recv_mb,
                'sent_max_mb': self.network_sent_max,
                'sent_min_mb': self.network_sent_min,
                'recv_max_mb': self.network_recv_max,
                'recv_min_mb': self.network_recv_min,
            },
            'process': {
                'pid': self.process_id,
                'threads': self.threads,
                'open_files': self.open_files,
                'connections': self.connections,
            }
        }
to_dict()

Konvertiert Metriken in Dictionary

Source code in toolboxv2/utils/system/types.py
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
def to_dict(self) -> Dict[str, Any]:
    """Konvertiert Metriken in Dictionary"""
    return {
        'uptime': {
            'seconds': self.uptime_seconds,
            'formatted': self.uptime_formatted,
        },
        'memory': {
            'current_mb': self.memory_current,
            'max_mb': self.memory_max,
            'min_mb': self.memory_min,
            'percent': self.memory_percent,
        },
        'cpu': {
            'current_percent': self.cpu_percent_current,
            'max_percent': self.cpu_percent_max,
            'min_percent': self.cpu_percent_min,
            'avg_percent': self.cpu_percent_avg,
            'time_seconds': self.cpu_time_seconds,
        },
        'disk': {
            'read_mb': self.disk_read_mb,
            'write_mb': self.disk_write_mb,
            'read_max_mb': self.disk_read_max,
            'read_min_mb': self.disk_read_min,
            'write_max_mb': self.disk_write_max,
            'write_min_mb': self.disk_write_min,
        },
        'network': {
            'sent_mb': self.network_sent_mb,
            'recv_mb': self.network_recv_mb,
            'sent_max_mb': self.network_sent_max,
            'sent_min_mb': self.network_sent_min,
            'recv_max_mb': self.network_recv_max,
            'recv_min_mb': self.network_recv_min,
        },
        'process': {
            'pid': self.process_id,
            'threads': self.threads,
            'open_files': self.open_files,
            'connections': self.connections,
        }
    }
Headers

Class representing HTTP headers with strongly typed common fields.

Source code in toolboxv2/utils/system/types.py
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
@dataclass
class Headers:
    """Class representing HTTP headers with strongly typed common fields."""
    # General Headers
    accept: None | str= None
    accept_charset: None | str= None
    accept_encoding: None | str= None
    accept_language: None | str= None
    accept_ranges: None | str= None
    access_control_allow_credentials: None | str= None
    access_control_allow_headers: None | str= None
    access_control_allow_methods: None | str= None
    access_control_allow_origin: None | str= None
    access_control_expose_headers: None | str= None
    access_control_max_age: None | str= None
    access_control_request_headers: None | str= None
    access_control_request_method: None | str= None
    age: None | str= None
    allow: None | str= None
    alt_svc: None | str= None
    authorization: None | str= None
    cache_control: None | str= None
    clear_site_data: None | str= None
    connection: None | str= None
    content_disposition: None | str= None
    content_encoding: None | str= None
    content_language: None | str= None
    content_length: None | str= None
    content_location: None | str= None
    content_range: None | str= None
    content_security_policy: None | str= None
    content_security_policy_report_only: None | str= None
    content_type: None | str= None
    cookie: None | str= None
    cross_origin_embedder_policy: None | str= None
    cross_origin_opener_policy: None | str= None
    cross_origin_resource_policy: None | str= None
    date: None | str= None
    device_memory: None | str= None
    digest: None | str= None
    dnt: None | str= None
    dpr: None | str= None
    etag: None | str= None
    expect: None | str= None
    expires: None | str= None
    feature_policy: None | str= None
    forwarded: None | str= None
    from_header: None | str= None  # 'from' is a Python keyword
    host: None | str= None
    if_match: None | str= None
    if_modified_since: None | str= None
    if_none_match: None | str= None
    if_range: None | str= None
    if_unmodified_since: None | str= None
    keep_alive: None | str= None
    large_allocation: None | str= None
    last_modified: None | str= None
    link: None | str= None
    location: None | str= None
    max_forwards: None | str= None
    origin: None | str= None
    pragma: None | str= None
    proxy_authenticate: None | str= None
    proxy_authorization: None | str= None
    public_key_pins: None | str= None
    public_key_pins_report_only: None | str= None
    range: None | str= None
    referer: None | str= None
    referrer_policy: None | str= None
    retry_after: None | str= None
    save_data: None | str= None
    sec_fetch_dest: None | str= None
    sec_fetch_mode: None | str= None
    sec_fetch_site: None | str= None
    sec_fetch_user: None | str= None
    sec_websocket_accept: None | str= None
    sec_websocket_extensions: None | str= None
    sec_websocket_key: None | str= None
    sec_websocket_protocol: None | str= None
    sec_websocket_version: None | str= None
    server: None | str= None
    server_timing: None | str= None
    service_worker_allowed: None | str= None
    set_cookie: None | str= None
    sourcemap: None | str= None
    strict_transport_security: None | str= None
    te: None | str= None
    timing_allow_origin: None | str= None
    tk: None | str= None
    trailer: None | str= None
    transfer_encoding: None | str= None
    upgrade: None | str= None
    upgrade_insecure_requests: None | str= None
    user_agent: None | str= None
    vary: None | str= None
    via: None | str= None
    warning: None | str= None
    www_authenticate: None | str= None
    x_content_type_options: None | str= None
    x_dns_prefetch_control: None | str= None
    x_forwarded_for: None | str= None
    x_forwarded_host: None | str= None
    x_forwarded_proto: None | str= None
    x_frame_options: None | str= None
    x_xss_protection: None | str= None

    # Browser-specific and custom headers
    sec_ch_ua: None | str= None
    sec_ch_ua_mobile: None | str= None
    sec_ch_ua_platform: None | str= None
    sec_ch_ua_arch: None | str= None
    sec_ch_ua_bitness: None | str= None
    sec_ch_ua_full_version: None | str= None
    sec_ch_ua_full_version_list: None | str= None
    sec_ch_ua_platform_version: None | str= None

    # HTMX specific headers
    hx_boosted: None | str= None
    hx_current_url: None | str= None
    hx_history_restore_request: None | str= None
    hx_prompt: None | str= None
    hx_request: None | str= None
    hx_target: None | str= None
    hx_trigger: None | str= None
    hx_trigger_name: None | str= None

    # Additional fields can be stored in extra_headers
    extra_headers: dict[str, str] = field(default_factory=dict)

    def __post_init__(self):
        """Convert header keys with hyphens to underscores for attribute access."""
        # Handle the 'from' header specifically since it's a Python keyword
        if 'from' in self.__dict__:
            self.from_header = self.__dict__.pop('from')

        # Store any attributes that weren't explicitly defined in extra_headers
        all_attrs = self.__annotations__.keys()
        for key in list(self.__dict__.keys()):
            if key not in all_attrs and key != "extra_headers":
                self.extra_headers[key.replace("_", "-")] = getattr(self, key)
                delattr(self, key)

    @classmethod
    def from_dict(cls, headers_dict: dict[str, str]) -> 'Headers':
        """Create a Headers instance from a dictionary."""
        # Convert header keys from hyphenated to underscore format for Python attributes
        processed_headers = {}
        extra_headers = {}

        for key, value in headers_dict.items():
            # Handle 'from' header specifically
            if key.lower() == 'from':
                processed_headers['from_header'] = value
                continue

            python_key = key.replace("-", "_").lower()
            if python_key in cls.__annotations__ and python_key != "extra_headers":
                processed_headers[python_key] = value
            else:
                extra_headers[key] = value

        return cls(**processed_headers, extra_headers=extra_headers)

    def to_dict(self) -> dict[str, str]:
        """Convert the Headers object back to a dictionary."""
        result = {}

        # Add regular attributes
        for key, value in self.__dict__.items():
            if key != "extra_headers" and value is not None:
                # Handle from_header specially
                if key == "from_header":
                    result["from"] = value
                else:
                    result[key.replace("_", "-")] = value

        # Add extra headers
        result.update(self.extra_headers)

        return result
__post_init__()

Convert header keys with hyphens to underscores for attribute access.

Source code in toolboxv2/utils/system/types.py
161
162
163
164
165
166
167
168
169
170
171
172
def __post_init__(self):
    """Convert header keys with hyphens to underscores for attribute access."""
    # Handle the 'from' header specifically since it's a Python keyword
    if 'from' in self.__dict__:
        self.from_header = self.__dict__.pop('from')

    # Store any attributes that weren't explicitly defined in extra_headers
    all_attrs = self.__annotations__.keys()
    for key in list(self.__dict__.keys()):
        if key not in all_attrs and key != "extra_headers":
            self.extra_headers[key.replace("_", "-")] = getattr(self, key)
            delattr(self, key)
from_dict(headers_dict) classmethod

Create a Headers instance from a dictionary.

Source code in toolboxv2/utils/system/types.py
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
@classmethod
def from_dict(cls, headers_dict: dict[str, str]) -> 'Headers':
    """Create a Headers instance from a dictionary."""
    # Convert header keys from hyphenated to underscore format for Python attributes
    processed_headers = {}
    extra_headers = {}

    for key, value in headers_dict.items():
        # Handle 'from' header specifically
        if key.lower() == 'from':
            processed_headers['from_header'] = value
            continue

        python_key = key.replace("-", "_").lower()
        if python_key in cls.__annotations__ and python_key != "extra_headers":
            processed_headers[python_key] = value
        else:
            extra_headers[key] = value

    return cls(**processed_headers, extra_headers=extra_headers)
to_dict()

Convert the Headers object back to a dictionary.

Source code in toolboxv2/utils/system/types.py
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
def to_dict(self) -> dict[str, str]:
    """Convert the Headers object back to a dictionary."""
    result = {}

    # Add regular attributes
    for key, value in self.__dict__.items():
        if key != "extra_headers" and value is not None:
            # Handle from_header specially
            if key == "from_header":
                result["from"] = value
            else:
                result[key.replace("_", "-")] = value

    # Add extra headers
    result.update(self.extra_headers)

    return result
MainToolType
Source code in toolboxv2/utils/system/types.py
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
class MainToolType:
    toolID: str
    app: A
    interface: ToolBoxInterfaces
    spec: str

    version: str
    tools: dict  # legacy
    name: str
    logger: logging
    color: str
    todo: Callable
    _on_exit: Callable
    stuf: bool
    config: dict
    user: U | None
    description: str

    @staticmethod
    def return_result(error: ToolBoxError = ToolBoxError.none,
                      exec_code: int = 0,
                      help_text: str = "",
                      data_info=None,
                      data=None,
                      data_to=None) -> Result:
        """proxi attr"""

    def load(self):
        """proxi attr"""

    def print(self, message, end="\n", **kwargs):
        """proxi attr"""

    def add_str_to_config(self, command):
        if len(command) != 2:
            self.logger.error('Invalid command must be key value')
            return False
        self.config[command[0]] = command[1]

    def webInstall(self, user_instance, construct_render) -> str:
        """"Returns a web installer for the given user instance and construct render template"""

    async def get_user(self, username: str) -> Result:
        return self.app.a_run_any(CLOUDM_AUTHMANAGER.GET_USER_BY_NAME, username=username, get_results=True)
load()

proxi attr

Source code in toolboxv2/utils/system/types.py
1309
1310
def load(self):
    """proxi attr"""
print(message, end='\n', **kwargs)

proxi attr

Source code in toolboxv2/utils/system/types.py
1312
1313
def print(self, message, end="\n", **kwargs):
    """proxi attr"""
return_result(error=ToolBoxError.none, exec_code=0, help_text='', data_info=None, data=None, data_to=None) staticmethod

proxi attr

Source code in toolboxv2/utils/system/types.py
1300
1301
1302
1303
1304
1305
1306
1307
@staticmethod
def return_result(error: ToolBoxError = ToolBoxError.none,
                  exec_code: int = 0,
                  help_text: str = "",
                  data_info=None,
                  data=None,
                  data_to=None) -> Result:
    """proxi attr"""
webInstall(user_instance, construct_render)

"Returns a web installer for the given user instance and construct render template

Source code in toolboxv2/utils/system/types.py
1321
1322
def webInstall(self, user_instance, construct_render) -> str:
    """"Returns a web installer for the given user instance and construct render template"""
Request

Class representing an HTTP request.

Source code in toolboxv2/utils/system/types.py
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
@dataclass
class Request:
    """Class representing an HTTP request."""
    content_type: str
    headers: Headers
    method: str
    path: str
    query_params: dict[str, Any] = field(default_factory=dict)
    form_data: dict[str, Any] | None = None
    body: Any | None = None

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'Request':
        """Create a Request instance from a dictionary."""
        headers = Headers.from_dict(data.get('headers', {}))

        # Extract other fields
        return cls(
            content_type=data.get('content_type', ''),
            headers=headers,
            method=data.get('method', ''),
            path=data.get('path', ''),
            query_params=data.get('query_params', {}),
            form_data=data.get('form_data'),
            body=data.get('body')
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert the Request object back to a dictionary."""
        result = {
            'content_type': self.content_type,
            'headers': self.headers.to_dict(),
            'method': self.method,
            'path': self.path,
            'query_params': self.query_params,
        }

        if self.form_data is not None:
            result['form_data'] = self.form_data

        if self.body is not None:
            result['body'] = self.body

        return result
from_dict(data) classmethod

Create a Request instance from a dictionary.

Source code in toolboxv2/utils/system/types.py
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'Request':
    """Create a Request instance from a dictionary."""
    headers = Headers.from_dict(data.get('headers', {}))

    # Extract other fields
    return cls(
        content_type=data.get('content_type', ''),
        headers=headers,
        method=data.get('method', ''),
        path=data.get('path', ''),
        query_params=data.get('query_params', {}),
        form_data=data.get('form_data'),
        body=data.get('body')
    )
to_dict()

Convert the Request object back to a dictionary.

Source code in toolboxv2/utils/system/types.py
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
def to_dict(self) -> dict[str, Any]:
    """Convert the Request object back to a dictionary."""
    result = {
        'content_type': self.content_type,
        'headers': self.headers.to_dict(),
        'method': self.method,
        'path': self.path,
        'query_params': self.query_params,
    }

    if self.form_data is not None:
        result['form_data'] = self.form_data

    if self.body is not None:
        result['body'] = self.body

    return result
RequestData

Main class representing the complete request data structure.

Source code in toolboxv2/utils/system/types.py
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
@dataclass
class RequestData:
    """Main class representing the complete request data structure."""
    request: Request
    session: Session
    session_id: str

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
        """Create a RequestData instance from a dictionary."""
        return cls(
            request=Request.from_dict(data.get('request', {})),
            session=Session.from_dict(data.get('session', {})),
            session_id=data.get('session_id', '')
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert the RequestData object back to a dictionary."""
        return {
            'request': self.request.to_dict(),
            'session': self.session.to_dict(),
            'session_id': self.session_id
        }

    def __getattr__(self, name: str) -> Any:
        """Delegate unknown attributes to the `request` object."""
        # Nur wenn das Attribut nicht direkt in RequestData existiert
        # und auch nicht `session` oder `session_id` ist
        if hasattr(self.request, name):
            return getattr(self.request, name)
        raise AttributeError(f"'RequestData' object has no attribute '{name}'")

    @classmethod
    def moc(cls):
        return cls(
            request=Request.from_dict({
                'content_type': 'application/x-www-form-urlencoded',
                'headers': {
                    'accept': '*/*',
                    'accept-encoding': 'gzip, deflate, br, zstd',
                    'accept-language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
                    'connection': 'keep-alive',
                    'content-length': '107',
                    'content-type': 'application/x-www-form-urlencoded',
                    'cookie': 'session=abc123',
                    'host': 'localhost:8080',
                    'hx-current-url': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'hx-request': 'true',
                    'hx-target': 'estimates-guest_1fc2c9',
                    'hx-trigger': 'config-form-guest_1fc2c9',
                    'origin': 'http://localhost:8080',
                    'referer': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'sec-ch-ua': '"Chromium";v="134", "Not:A-Brand";v="24", "Google Chrome";v="134"',
                    'sec-ch-ua-mobile': '?0',
                    'sec-ch-ua-platform': '"Windows"',
                    'sec-fetch-dest': 'empty',
                    'sec-fetch-mode': 'cors',
                    'sec-fetch-site': 'same-origin',
                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
                },
                'method': 'POST',
                'path': '/api/TruthSeeker/update_estimates',
                'query_params': {},
                'form_data': {
                    'param1': 'value1',
                    'param2': 'value2'
                }
            }),
            session=Session.from_dict({
                'SiID': '29a2e258e18252e2afd5ff943523f09c82f1bb9adfe382a6f33fc6a8381de898',
                'level': '1',
                'spec': '74eed1c8de06886842e235486c3c2fd6bcd60586998ac5beb87f13c0d1750e1d',
                'user_name': 'root',
                'custom_field': 'custom_value'
            }),
            session_id='0x29dd1ac0d1e30d3f'
        )
__getattr__(name)

Delegate unknown attributes to the request object.

Source code in toolboxv2/utils/system/types.py
326
327
328
329
330
331
332
def __getattr__(self, name: str) -> Any:
    """Delegate unknown attributes to the `request` object."""
    # Nur wenn das Attribut nicht direkt in RequestData existiert
    # und auch nicht `session` oder `session_id` ist
    if hasattr(self.request, name):
        return getattr(self.request, name)
    raise AttributeError(f"'RequestData' object has no attribute '{name}'")
from_dict(data) classmethod

Create a RequestData instance from a dictionary.

Source code in toolboxv2/utils/system/types.py
309
310
311
312
313
314
315
316
@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
    """Create a RequestData instance from a dictionary."""
    return cls(
        request=Request.from_dict(data.get('request', {})),
        session=Session.from_dict(data.get('session', {})),
        session_id=data.get('session_id', '')
    )
to_dict()

Convert the RequestData object back to a dictionary.

Source code in toolboxv2/utils/system/types.py
318
319
320
321
322
323
324
def to_dict(self) -> dict[str, Any]:
    """Convert the RequestData object back to a dictionary."""
    return {
        'request': self.request.to_dict(),
        'session': self.session.to_dict(),
        'session_id': self.session_id
    }
Result
Source code in toolboxv2/utils/system/types.py
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task
__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py
643
644
645
646
647
648
649
650
651
652
653
def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult
binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)
cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py
738
739
740
741
742
743
744
745
746
747
748
def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result
file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name Type Description Default
data

File data as bytes or base64 string

required
filename

Name of the file for download

required
content_type

MIME type of the file (auto-detected if None)

None
info

Response info text

'OK'
interface

Target interface

remote

Returns:

Type Description

Result object configured for file download

Source code in toolboxv2/utils/system/types.py
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)
get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py
750
751
752
def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type
is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py
754
755
756
def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None
json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py
970
971
972
973
974
975
976
977
978
979
980
981
982
983
@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)
redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)
sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name Type Description Default
stream_generator Any

A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.

required
info str

Optional help text for the Result.

'OK'
interface ToolBoxInterfaces

Optional ToolBoxInterface to target.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional cleanup function to run when the stream ends or is cancelled.

None
#http_headers

Optional dictionary of custom HTTP headers for the SSE response.

required

Returns:

Type Description

A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )
stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name Type Description Default
stream_generator Any

Any stream source (async generator, sync generator, iterable, or single item).

required
content_type str

Content-Type header (default: text/event-stream for SSE).

'text/event-stream'
headers dict | None

Additional HTTP headers for the response.

None
info str

Help text for the result.

'OK'
interface ToolBoxInterfaces

Interface to send data to.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional function for cleanup.

None

Returns:

Type Description

A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)
text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)
typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py
667
668
669
670
671
672
673
674
675
676
async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data
typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py
655
656
657
658
659
660
661
662
663
664
665
def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data
typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance
typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py
705
706
707
708
709
710
711
712
713
714
715
716
@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance
SSEGenerator

Production-ready SSE generator that converts any data source to properly formatted Server-Sent Events compatible with browsers.

Source code in toolboxv2/utils/system/types.py
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
class SSEGenerator:
    """
    Production-ready SSE generator that converts any data source to
    properly formatted Server-Sent Events compatible with browsers.
    """

    @staticmethod
    def format_sse_event(data: Any) -> str:
        """Format any data as a proper SSE event message."""
        # Already formatted as SSE
        if isinstance(data, str) and (data.startswith('data:') or data.startswith('event:')) and '\n\n' in data:
            return data

        # Handle bytes (binary data)
        if isinstance(data, bytes):
            try:
                # Try to decode as UTF-8 first
                decoded_data_str = data.decode('utf-8')
                # If decoding works, treat it as a string for further processing
                # This allows binary data that is valid UTF-8 JSON to be processed as JSON.
                data = decoded_data_str
            except UnicodeDecodeError:
                # Binary data that is not UTF-8, encode as base64
                b64_data = base64.b64encode(data).decode('utf-8')
                return f"event: binary\ndata: {b64_data}\n\n"

        # Convert non-string objects (that are not already bytes) to JSON string
        # If data was bytes and successfully decoded to UTF-8 string, it will be processed here.
        original_data_type_was_complex = False
        if not isinstance(data, str):
            original_data_type_was_complex = True
            try:
                data_str = json.dumps(data)
            except Exception:
                data_str = str(data)  # Fallback to string representation
        else:
            data_str = data  # data is already a string

        # Handle JSON data with special event formatting
        # data_str now holds the string representation (either original string or JSON string)
        if data_str.strip().startswith('{'):
            try:
                json_data = json.loads(data_str)
                if isinstance(json_data, dict) and 'event' in json_data:
                    event_type = json_data['event']
                    event_id = json_data.get('id', None)  # Use None to distinguish from empty string

                    # Determine the actual data payload for the SSE 'data:' field
                    # If 'data' key exists in json_data, use its content.
                    # Otherwise, use the original data_str (which is the JSON of json_data).
                    if 'data' in json_data:
                        payload_content = json_data['data']
                        # If payload_content is complex, re-serialize it to JSON string
                        if isinstance(payload_content, dict | list):
                            sse_data_field = json.dumps(payload_content)
                        else:  # Simple type (string, number, bool)
                            sse_data_field = str(payload_content)
                    else:
                        # If original data was complex (e.g. dict) and became json_data,
                        # and no 'data' key in it, then use the full json_data as payload.
                        # If original data was a simple string that happened to be JSON parsable
                        # but without 'event' key, it would have been handled by "Regular JSON without event"
                        # or "Plain text" later.
                        # This path implies original data was a dict with 'event' key.
                        sse_data_field = data_str

                    sse_lines = []
                    if event_type:  # Should always be true here
                        sse_lines.append(f"event: {event_type}")
                    if event_id is not None:  # Check for None, allow empty string id
                        sse_lines.append(f"id: {event_id}")

                    # Handle multi-line data for the data field
                    for line in sse_data_field.splitlines():
                        sse_lines.append(f"data: {line}")

                    return "\n".join(sse_lines) + "\n\n"
                else:
                    # Regular JSON without special 'event' key
                    sse_lines = []
                    for line in data_str.splitlines():
                        sse_lines.append(f"data: {line}")
                    return "\n".join(sse_lines) + "\n\n"
            except json.JSONDecodeError:
                # Not valid JSON, treat as plain text
                sse_lines = []
                for line in data_str.splitlines():
                    sse_lines.append(f"data: {line}")
                return "\n".join(sse_lines) + "\n\n"
        else:
            # Plain text
            sse_lines = []
            for line in data_str.splitlines():
                sse_lines.append(f"data: {line}")
            return "\n".join(sse_lines) + "\n\n"

    @classmethod
    async def wrap_sync_generator(cls, generator):
        """Convert a synchronous generator to an async generator."""
        for item in generator:
            yield item
            # Allow other tasks to run
            await asyncio.sleep(0)

    @classmethod
    async def create_sse_stream(
        cls,
        source: Any,  # Changed from positional arg to keyword for clarity in Result.stream
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None
    ) -> AsyncGenerator[str, None]:
        """
        Convert any source to a properly formatted SSE stream.

        Args:
            source: Can be async generator, sync generator, iterable, or a single item.
            cleanup_func: Optional function to call when the stream ends or is cancelled.
                          Can be a synchronous function, async function, or async generator.

        Yields:
            Properly formatted SSE messages (strings).
        """
        # Send stream start event
        # This structure ensures data field contains {"id":"0"}
        yield cls.format_sse_event({"event": "stream_start", "data": {"id": "0"}})

        try:
            # Handle different types of sources
            if inspect.isasyncgen(source):
                # Source is already an async generator
                async for item in source:
                    yield cls.format_sse_event(item)
            elif inspect.isgenerator(source) or (not isinstance(source, str) and hasattr(source, '__iter__')):
                # Source is a sync generator or iterable (but not a string)
                # Strings are iterable but should be treated as single items unless explicitly made a generator
                async for item in cls.wrap_sync_generator(source):
                    yield cls.format_sse_event(item)
            else:
                # Single item (including strings)
                yield cls.format_sse_event(source)
        except asyncio.CancelledError:
            # Client disconnected
            yield cls.format_sse_event({"event": "cancelled", "data": {"id": "cancelled"}})
            raise
        except Exception as e:
            # Error in stream
            error_info = {
                "event": "error",
                "data": {  # Ensure payload is under 'data' key for the new format_sse_event logic
                    "message": str(e),
                    "traceback": traceback.format_exc()
                }
            }
            yield cls.format_sse_event(error_info)
        finally:
            # Always send end event
            yield cls.format_sse_event({"event": "stream_end", "data": {"id": "final"}})

            # Execute cleanup function if provided
            if cleanup_func:
                try:
                    if inspect.iscoroutinefunction(cleanup_func):  # Check if it's an async def function
                        await cleanup_func()
                    elif inspect.isasyncgenfunction(cleanup_func) or inspect.isasyncgen(
                        cleanup_func):  # Check if it's an async def generator function or already an async generator
                        # If it's a function, call it to get the generator
                        gen_to_exhaust = cleanup_func() if inspect.isasyncgenfunction(cleanup_func) else cleanup_func
                        async for _ in gen_to_exhaust:
                            pass  # Exhaust the generator to ensure cleanup completes
                    else:
                        # Synchronous function
                        cleanup_func()
                except Exception as e:
                    # Log cleanup errors but don't propagate them to client
                    error_info_cleanup = {
                        "event": "cleanup_error",
                        "data": {  # Ensure payload is under 'data' key
                            "message": str(e),
                            "traceback": traceback.format_exc()
                        }
                    }
                    # We can't yield here as the stream is already closing/closed.
                    # Instead, log the error.
                    # In a real app, use a proper logger.
                    print(f"SSE cleanup error: {cls.format_sse_event(error_info_cleanup)}", flush=True)
create_sse_stream(source, cleanup_func=None) async classmethod

Convert any source to a properly formatted SSE stream.

Parameters:

Name Type Description Default
source Any

Can be async generator, sync generator, iterable, or a single item.

required
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional function to call when the stream ends or is cancelled. Can be a synchronous function, async function, or async generator.

None

Yields:

Type Description
AsyncGenerator[str, None]

Properly formatted SSE messages (strings).

Source code in toolboxv2/utils/system/types.py
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
@classmethod
async def create_sse_stream(
    cls,
    source: Any,  # Changed from positional arg to keyword for clarity in Result.stream
    cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None
) -> AsyncGenerator[str, None]:
    """
    Convert any source to a properly formatted SSE stream.

    Args:
        source: Can be async generator, sync generator, iterable, or a single item.
        cleanup_func: Optional function to call when the stream ends or is cancelled.
                      Can be a synchronous function, async function, or async generator.

    Yields:
        Properly formatted SSE messages (strings).
    """
    # Send stream start event
    # This structure ensures data field contains {"id":"0"}
    yield cls.format_sse_event({"event": "stream_start", "data": {"id": "0"}})

    try:
        # Handle different types of sources
        if inspect.isasyncgen(source):
            # Source is already an async generator
            async for item in source:
                yield cls.format_sse_event(item)
        elif inspect.isgenerator(source) or (not isinstance(source, str) and hasattr(source, '__iter__')):
            # Source is a sync generator or iterable (but not a string)
            # Strings are iterable but should be treated as single items unless explicitly made a generator
            async for item in cls.wrap_sync_generator(source):
                yield cls.format_sse_event(item)
        else:
            # Single item (including strings)
            yield cls.format_sse_event(source)
    except asyncio.CancelledError:
        # Client disconnected
        yield cls.format_sse_event({"event": "cancelled", "data": {"id": "cancelled"}})
        raise
    except Exception as e:
        # Error in stream
        error_info = {
            "event": "error",
            "data": {  # Ensure payload is under 'data' key for the new format_sse_event logic
                "message": str(e),
                "traceback": traceback.format_exc()
            }
        }
        yield cls.format_sse_event(error_info)
    finally:
        # Always send end event
        yield cls.format_sse_event({"event": "stream_end", "data": {"id": "final"}})

        # Execute cleanup function if provided
        if cleanup_func:
            try:
                if inspect.iscoroutinefunction(cleanup_func):  # Check if it's an async def function
                    await cleanup_func()
                elif inspect.isasyncgenfunction(cleanup_func) or inspect.isasyncgen(
                    cleanup_func):  # Check if it's an async def generator function or already an async generator
                    # If it's a function, call it to get the generator
                    gen_to_exhaust = cleanup_func() if inspect.isasyncgenfunction(cleanup_func) else cleanup_func
                    async for _ in gen_to_exhaust:
                        pass  # Exhaust the generator to ensure cleanup completes
                else:
                    # Synchronous function
                    cleanup_func()
            except Exception as e:
                # Log cleanup errors but don't propagate them to client
                error_info_cleanup = {
                    "event": "cleanup_error",
                    "data": {  # Ensure payload is under 'data' key
                        "message": str(e),
                        "traceback": traceback.format_exc()
                    }
                }
                # We can't yield here as the stream is already closing/closed.
                # Instead, log the error.
                # In a real app, use a proper logger.
                print(f"SSE cleanup error: {cls.format_sse_event(error_info_cleanup)}", flush=True)
format_sse_event(data) staticmethod

Format any data as a proper SSE event message.

Source code in toolboxv2/utils/system/types.py
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
@staticmethod
def format_sse_event(data: Any) -> str:
    """Format any data as a proper SSE event message."""
    # Already formatted as SSE
    if isinstance(data, str) and (data.startswith('data:') or data.startswith('event:')) and '\n\n' in data:
        return data

    # Handle bytes (binary data)
    if isinstance(data, bytes):
        try:
            # Try to decode as UTF-8 first
            decoded_data_str = data.decode('utf-8')
            # If decoding works, treat it as a string for further processing
            # This allows binary data that is valid UTF-8 JSON to be processed as JSON.
            data = decoded_data_str
        except UnicodeDecodeError:
            # Binary data that is not UTF-8, encode as base64
            b64_data = base64.b64encode(data).decode('utf-8')
            return f"event: binary\ndata: {b64_data}\n\n"

    # Convert non-string objects (that are not already bytes) to JSON string
    # If data was bytes and successfully decoded to UTF-8 string, it will be processed here.
    original_data_type_was_complex = False
    if not isinstance(data, str):
        original_data_type_was_complex = True
        try:
            data_str = json.dumps(data)
        except Exception:
            data_str = str(data)  # Fallback to string representation
    else:
        data_str = data  # data is already a string

    # Handle JSON data with special event formatting
    # data_str now holds the string representation (either original string or JSON string)
    if data_str.strip().startswith('{'):
        try:
            json_data = json.loads(data_str)
            if isinstance(json_data, dict) and 'event' in json_data:
                event_type = json_data['event']
                event_id = json_data.get('id', None)  # Use None to distinguish from empty string

                # Determine the actual data payload for the SSE 'data:' field
                # If 'data' key exists in json_data, use its content.
                # Otherwise, use the original data_str (which is the JSON of json_data).
                if 'data' in json_data:
                    payload_content = json_data['data']
                    # If payload_content is complex, re-serialize it to JSON string
                    if isinstance(payload_content, dict | list):
                        sse_data_field = json.dumps(payload_content)
                    else:  # Simple type (string, number, bool)
                        sse_data_field = str(payload_content)
                else:
                    # If original data was complex (e.g. dict) and became json_data,
                    # and no 'data' key in it, then use the full json_data as payload.
                    # If original data was a simple string that happened to be JSON parsable
                    # but without 'event' key, it would have been handled by "Regular JSON without event"
                    # or "Plain text" later.
                    # This path implies original data was a dict with 'event' key.
                    sse_data_field = data_str

                sse_lines = []
                if event_type:  # Should always be true here
                    sse_lines.append(f"event: {event_type}")
                if event_id is not None:  # Check for None, allow empty string id
                    sse_lines.append(f"id: {event_id}")

                # Handle multi-line data for the data field
                for line in sse_data_field.splitlines():
                    sse_lines.append(f"data: {line}")

                return "\n".join(sse_lines) + "\n\n"
            else:
                # Regular JSON without special 'event' key
                sse_lines = []
                for line in data_str.splitlines():
                    sse_lines.append(f"data: {line}")
                return "\n".join(sse_lines) + "\n\n"
        except json.JSONDecodeError:
            # Not valid JSON, treat as plain text
            sse_lines = []
            for line in data_str.splitlines():
                sse_lines.append(f"data: {line}")
            return "\n".join(sse_lines) + "\n\n"
    else:
        # Plain text
        sse_lines = []
        for line in data_str.splitlines():
            sse_lines.append(f"data: {line}")
        return "\n".join(sse_lines) + "\n\n"
wrap_sync_generator(generator) async classmethod

Convert a synchronous generator to an async generator.

Source code in toolboxv2/utils/system/types.py
2574
2575
2576
2577
2578
2579
2580
@classmethod
async def wrap_sync_generator(cls, generator):
    """Convert a synchronous generator to an async generator."""
    for item in generator:
        yield item
        # Allow other tasks to run
        await asyncio.sleep(0)
Session

Class representing a session.

Source code in toolboxv2/utils/system/types.py
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
@dataclass
class Session:
    """Class representing a session."""
    SiID: str
    level: str
    spec: str
    user_name: str
    # Allow for additional fields
    extra_data: dict[str, Any] = field(default_factory=dict)

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'Session':
        """Create a Session instance from a dictionary with default values."""
        known_fields = {
            'SiID': data.get('SiID', '#0'),
            'level': data.get('level', -1),
            'spec': data.get('spec', 'app'),
            'user_name': data.get('user_name', 'anonymous'),
        }

        extra_data = {k: v for k, v in data.items() if k not in known_fields}
        return cls(**known_fields, extra_data=extra_data)

    def to_dict(self) -> dict[str, Any]:
        """Convert the Session object back to a dictionary."""
        result = {
            'SiID': self.SiID,
            'level': self.level,
            'spec': self.spec,
            'user_name': self.user_name,
        }

        # Add extra data
        result.update(self.extra_data)

        return result

    @property
    def valid(self):
        return int(self.level) > 0
from_dict(data) classmethod

Create a Session instance from a dictionary with default values.

Source code in toolboxv2/utils/system/types.py
270
271
272
273
274
275
276
277
278
279
280
281
@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'Session':
    """Create a Session instance from a dictionary with default values."""
    known_fields = {
        'SiID': data.get('SiID', '#0'),
        'level': data.get('level', -1),
        'spec': data.get('spec', 'app'),
        'user_name': data.get('user_name', 'anonymous'),
    }

    extra_data = {k: v for k, v in data.items() if k not in known_fields}
    return cls(**known_fields, extra_data=extra_data)
to_dict()

Convert the Session object back to a dictionary.

Source code in toolboxv2/utils/system/types.py
283
284
285
286
287
288
289
290
291
292
293
294
295
def to_dict(self) -> dict[str, Any]:
    """Convert the Session object back to a dictionary."""
    result = {
        'SiID': self.SiID,
        'level': self.level,
        'spec': self.spec,
        'user_name': self.user_name,
    }

    # Add extra data
    result.update(self.extra_data)

    return result
parse_request_data(data)

Parse the incoming request data into a strongly typed structure.

Source code in toolboxv2/utils/system/types.py
382
383
384
def parse_request_data(data: dict[str, Any]) -> RequestData:
    """Parse the incoming request data into a strongly typed structure."""
    return RequestData.from_dict(data)

tbx

install_support

Complete TB Language Setup - Build executable - Setup file associations - Install VS Code extension - Install PyCharm plugin

TBSetup

Complete TB Language setup manager

Source code in toolboxv2/utils/tbx/install_support.py
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
class TBSetup:
    """Complete TB Language setup manager"""

    def __init__(self):
        self.root = Path(__file__).parent
        self.system = platform.system()

    def setup_all(self):
        """Run complete setup"""
        print("═" * 70)
        print("  TB Language - Complete Setup")
        print("═" * 70)
        print()

        success = True

        # Step 1: Build
        if not self.build_executable():
            print("❌ Build failed!")
            return False

        # Step 2: System integration
        if not self.setup_system_integration():
            print("⚠️  System integration failed (optional)")
            success = False

        # Step 3: VS Code extension
        if not self.setup_vscode():
            print("⚠️  VS Code extension setup failed (optional)")
            success = False

        # Step 4: PyCharm plugin
        if not self.setup_pycharm():
            print("⚠️  PyCharm plugin setup failed (optional)")
            success = False

        print()
        print("═" * 70)
        if success:
            print("  ✓ Setup Complete!")
        else:
            print("  ⚠️  Setup completed with warnings")
        print("═" * 70)
        print()
        print("Next steps:")
        print("  1. Restart PyCharm and VS Code (if open)")
        print("  2. Create a test file: test.tbx")
        print("  3. Run it: tb run test.tbx")
        print("  4. Or double-click test.tbx to run")
        print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
        print()

        return success

    def build_executable(self):
        """Step 1: Build TB Language"""
        print("Step 1/4: Building TB Language...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
            "build"
        ])

        if result.returncode != 0:
            return False

        print("✓ Build successful")
        print()
        return True

    def setup_system_integration(self):
        """Step 2: System integration"""
        print("Step 2/4: Setting up system integration...")
        print("-" * 70)

        result = subprocess.run([
            sys.executable,
            str(self.root / "toolbox-exec" / "tb_setup.py"),
            "install"
        ])

        print()
        return result.returncode == 0

    def setup_vscode(self):
        """Step 3: VS Code extension"""
        print("Step 3/4: Installing VS Code extension...")
        print("-" * 70)

        vscode_ext = self.root / "tb-lang-vscode"
        if not vscode_ext.exists():
            print("⚠️  VS Code extension directory not found")
            print()
            return False

        try:
            # Check if npm is available
            subprocess.run(["npm", "--version"],
                           capture_output=True, check=True)

            # Install dependencies
            print("  Installing npm dependencies...")
            subprocess.run(["npm", "install"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Compile TypeScript
            print("  Compiling TypeScript...")
            subprocess.run(["npm", "run", "compile"],
                           cwd=vscode_ext,
                           capture_output=True,
                           check=True)

            # Try to install to VS Code
            print("  Installing to VS Code...")
            result = subprocess.run([
                "code", "--install-extension", str(vscode_ext.resolve())
            ], capture_output=True)

            if result.returncode == 0:
                print("✓ VS Code extension installed")
                print()
                return True
            else:
                print("⚠️  Could not auto-install to VS Code")
                print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
                print()
                return False

        except FileNotFoundError as e:
            print(f"⚠️  Tool not found: {e}")
            print("   npm: https://nodejs.org/")
            print("   VS Code: https://code.visualstudio.com/")
            print()
            return False
        except subprocess.CalledProcessError as e:
            print(f"⚠️  Command failed: {e}")
            print()
            return False

    def setup_pycharm(self):
        """Step 4: PyCharm plugin"""
        print("Step 4/4: Installing PyCharm plugin...")
        print("-" * 70)

        pycharm_plugin = self.root / "tb-lang-pycharm"
        if not pycharm_plugin.exists():
            print("⚠️  PyCharm plugin directory not found")
            print("   Creating plugin structure...")
            if not self.create_pycharm_plugin():
                print()
                return False

        try:
            # Build plugin JAR
            print("  Building PyCharm plugin...")
            if not self.build_pycharm_plugin():
                print("⚠️  Plugin build failed")
                print()
                return False

            # Install to PyCharm
            print("  Installing to PyCharm...")
            if not self.install_pycharm_plugin():
                print("⚠️  Auto-install failed")
                print()
                return False

            print("✓ PyCharm plugin installed")
            print("  Please restart PyCharm to activate the plugin")
            print()
            return True

        except Exception as e:
            print(f"⚠️  Error: {e}")
            print()
            return False

    def create_pycharm_plugin(self):
        """Create PyCharm plugin structure"""
        plugin_dir = self.root / "tb-lang-pycharm"
        plugin_dir.mkdir(exist_ok=True)

        # Create directory structure
        (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
        (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

        return True

    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False

    def install_pycharm_plugin(self):
        """Install plugin to PyCharm"""
        plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

        if not plugin_jar.exists():
            print("  Plugin JAR not found")
            return False

        # Find PyCharm config directory
        pycharm_dirs = self.find_pycharm_config_dirs()

        if not pycharm_dirs:
            print("  PyCharm installation not found")
            print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
            return False

        # Install to all found PyCharm installations
        installed = False
        for config_dir in pycharm_dirs:
            plugins_dir = config_dir / "plugins"
            plugins_dir.mkdir(exist_ok=True)

            dest = plugins_dir / "tb-language.jar"
            shutil.copy(plugin_jar, dest)
            print(f"  ✓ Installed to: {dest}")
            installed = True

        return installed

    def find_pycharm_config_dirs(self):
        """Find PyCharm config directories"""
        config_dirs = []
        home = Path.home()

        if self.system == "Windows":
            # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
            base = home / "AppData" / "Roaming" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        elif self.system == "Linux":
            # Linux: ~/.config/JetBrains/PyCharm*
            base = home / ".config" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

            # Also check old location
            old_base = home / ".PyCharm*"
            config_dirs.extend(home.glob(".PyCharm*"))

        elif self.system == "Darwin":
            # macOS: ~/Library/Application Support/JetBrains/PyCharm*
            base = home / "Library" / "Application Support" / "JetBrains"
            if base.exists():
                config_dirs.extend(base.glob("PyCharm*"))

        return [d for d in config_dirs if d.is_dir()]
build_executable()

Step 1: Build TB Language

Source code in toolboxv2/utils/tbx/install_support.py
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def build_executable(self):
    """Step 1: Build TB Language"""
    print("Step 1/4: Building TB Language...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_lang_cli.py"),
        "build"
    ])

    if result.returncode != 0:
        return False

    print("✓ Build successful")
    print()
    return True
build_pycharm_plugin()

Build PyCharm plugin JAR

Source code in toolboxv2/utils/tbx/install_support.py
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
    def build_pycharm_plugin(self):
        """Build PyCharm plugin JAR"""
        plugin_dir = self.root / "tb-lang-pycharm"
        build_script = plugin_dir / "build_plugin.py"

        if not build_script.exists():
            # Create build script
            build_script.write_text('''#!/usr/bin/env python3
import zipfile
from pathlib import Path

plugin_dir = Path(__file__).parent
output_jar = plugin_dir / "tb-language.jar"

with zipfile.ZipFile(output_jar, 'w', zipfile.ZIP_DEFLATED) as jar:
    # Add plugin.xml
    plugin_xml = plugin_dir / "src" / "main" / "resources" / "META-INF" / "plugin.xml"
    if plugin_xml.exists():
        jar.write(plugin_xml, "META-INF/plugin.xml")

    # Add file type definition
    file_type = plugin_dir / "src" / "main" / "resources" / "fileTypes" / "TB.xml"
    if file_type.exists():
        jar.write(file_type, "fileTypes/TB.xml")

print(f"✓ Plugin built: {output_jar}")
''')
            build_script.chmod(0o755)

        # Run build script
        result = subprocess.run([sys.executable, str(build_script)],
                                capture_output=True, text=True)

        if result.returncode == 0:
            print(f"  {result.stdout.strip()}")
            return True
        else:
            print(f"  Build error: {result.stderr}")
            return False
create_pycharm_plugin()

Create PyCharm plugin structure

Source code in toolboxv2/utils/tbx/install_support.py
199
200
201
202
203
204
205
206
207
208
def create_pycharm_plugin(self):
    """Create PyCharm plugin structure"""
    plugin_dir = self.root / "tb-lang-pycharm"
    plugin_dir.mkdir(exist_ok=True)

    # Create directory structure
    (plugin_dir / "src" / "main" / "resources" / "fileTypes").mkdir(parents=True, exist_ok=True)
    (plugin_dir / "src" / "main" / "resources" / "META-INF").mkdir(parents=True, exist_ok=True)

    return True
find_pycharm_config_dirs()

Find PyCharm config directories

Source code in toolboxv2/utils/tbx/install_support.py
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
def find_pycharm_config_dirs(self):
    """Find PyCharm config directories"""
    config_dirs = []
    home = Path.home()

    if self.system == "Windows":
        # Windows: C:\Users\<user>\AppData\Roaming\JetBrains\PyCharm*
        base = home / "AppData" / "Roaming" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    elif self.system == "Linux":
        # Linux: ~/.config/JetBrains/PyCharm*
        base = home / ".config" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

        # Also check old location
        old_base = home / ".PyCharm*"
        config_dirs.extend(home.glob(".PyCharm*"))

    elif self.system == "Darwin":
        # macOS: ~/Library/Application Support/JetBrains/PyCharm*
        base = home / "Library" / "Application Support" / "JetBrains"
        if base.exists():
            config_dirs.extend(base.glob("PyCharm*"))

    return [d for d in config_dirs if d.is_dir()]
install_pycharm_plugin()

Install plugin to PyCharm

Source code in toolboxv2/utils/tbx/install_support.py
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
def install_pycharm_plugin(self):
    """Install plugin to PyCharm"""
    plugin_jar = self.root / "tb-lang-pycharm" / "tb-language.jar"

    if not plugin_jar.exists():
        print("  Plugin JAR not found")
        return False

    # Find PyCharm config directory
    pycharm_dirs = self.find_pycharm_config_dirs()

    if not pycharm_dirs:
        print("  PyCharm installation not found")
        print(f"  Manual install: Copy {plugin_jar} to PyCharm plugins directory")
        return False

    # Install to all found PyCharm installations
    installed = False
    for config_dir in pycharm_dirs:
        plugins_dir = config_dir / "plugins"
        plugins_dir.mkdir(exist_ok=True)

        dest = plugins_dir / "tb-language.jar"
        shutil.copy(plugin_jar, dest)
        print(f"  ✓ Installed to: {dest}")
        installed = True

    return installed
setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/install_support.py
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def setup_all(self):
    """Run complete setup"""
    print("═" * 70)
    print("  TB Language - Complete Setup")
    print("═" * 70)
    print()

    success = True

    # Step 1: Build
    if not self.build_executable():
        print("❌ Build failed!")
        return False

    # Step 2: System integration
    if not self.setup_system_integration():
        print("⚠️  System integration failed (optional)")
        success = False

    # Step 3: VS Code extension
    if not self.setup_vscode():
        print("⚠️  VS Code extension setup failed (optional)")
        success = False

    # Step 4: PyCharm plugin
    if not self.setup_pycharm():
        print("⚠️  PyCharm plugin setup failed (optional)")
        success = False

    print()
    print("═" * 70)
    if success:
        print("  ✓ Setup Complete!")
    else:
        print("  ⚠️  Setup completed with warnings")
    print("═" * 70)
    print()
    print("Next steps:")
    print("  1. Restart PyCharm and VS Code (if open)")
    print("  2. Create a test file: test.tbx")
    print("  3. Run it: tb run test.tbx")
    print("  4. Or double-click test.tbx to run")
    print("  5. Open .tbx files in PyCharm/VS Code for syntax highlighting")
    print()

    return success
setup_pycharm()

Step 4: PyCharm plugin

Source code in toolboxv2/utils/tbx/install_support.py
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
def setup_pycharm(self):
    """Step 4: PyCharm plugin"""
    print("Step 4/4: Installing PyCharm plugin...")
    print("-" * 70)

    pycharm_plugin = self.root / "tb-lang-pycharm"
    if not pycharm_plugin.exists():
        print("⚠️  PyCharm plugin directory not found")
        print("   Creating plugin structure...")
        if not self.create_pycharm_plugin():
            print()
            return False

    try:
        # Build plugin JAR
        print("  Building PyCharm plugin...")
        if not self.build_pycharm_plugin():
            print("⚠️  Plugin build failed")
            print()
            return False

        # Install to PyCharm
        print("  Installing to PyCharm...")
        if not self.install_pycharm_plugin():
            print("⚠️  Auto-install failed")
            print()
            return False

        print("✓ PyCharm plugin installed")
        print("  Please restart PyCharm to activate the plugin")
        print()
        return True

    except Exception as e:
        print(f"⚠️  Error: {e}")
        print()
        return False
setup_system_integration()

Step 2: System integration

Source code in toolboxv2/utils/tbx/install_support.py
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
def setup_system_integration(self):
    """Step 2: System integration"""
    print("Step 2/4: Setting up system integration...")
    print("-" * 70)

    result = subprocess.run([
        sys.executable,
        str(self.root / "toolbox-exec" / "tb_setup.py"),
        "install"
    ])

    print()
    return result.returncode == 0
setup_vscode()

Step 3: VS Code extension

Source code in toolboxv2/utils/tbx/install_support.py
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
def setup_vscode(self):
    """Step 3: VS Code extension"""
    print("Step 3/4: Installing VS Code extension...")
    print("-" * 70)

    vscode_ext = self.root / "tb-lang-vscode"
    if not vscode_ext.exists():
        print("⚠️  VS Code extension directory not found")
        print()
        return False

    try:
        # Check if npm is available
        subprocess.run(["npm", "--version"],
                       capture_output=True, check=True)

        # Install dependencies
        print("  Installing npm dependencies...")
        subprocess.run(["npm", "install"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Compile TypeScript
        print("  Compiling TypeScript...")
        subprocess.run(["npm", "run", "compile"],
                       cwd=vscode_ext,
                       capture_output=True,
                       check=True)

        # Try to install to VS Code
        print("  Installing to VS Code...")
        result = subprocess.run([
            "code", "--install-extension", str(vscode_ext.resolve())
        ], capture_output=True)

        if result.returncode == 0:
            print("✓ VS Code extension installed")
            print()
            return True
        else:
            print("⚠️  Could not auto-install to VS Code")
            print(f"   Manual install: code --install-extension {vscode_ext.resolve()}")
            print()
            return False

    except FileNotFoundError as e:
        print(f"⚠️  Tool not found: {e}")
        print("   npm: https://nodejs.org/")
        print("   VS Code: https://code.visualstudio.com/")
        print()
        return False
    except subprocess.CalledProcessError as e:
        print(f"⚠️  Command failed: {e}")
        print()
        return False
main()

Main entry point

Source code in toolboxv2/utils/tbx/install_support.py
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language Complete Setup"
    )
    parser.add_argument('--skip-build', action='store_true',
                        help='Skip building the executable')
    parser.add_argument('--skip-system', action='store_true',
                        help='Skip system integration')
    parser.add_argument('--skip-vscode', action='store_true',
                        help='Skip VS Code extension')
    parser.add_argument('--skip-pycharm', action='store_true',
                        help='Skip PyCharm plugin')
    parser.add_argument('--pycharm-only', action='store_true',
                        help='Only setup PyCharm plugin')

    args = parser.parse_args()

    setup = TBSetup()

    if args.pycharm_only:
        success = setup.setup_pycharm()
    else:
        # Full setup with skip options
        success = True

        if not args.skip_build:
            success = setup.build_executable() and success

        if not args.skip_system:
            setup.setup_system_integration()

        if not args.skip_vscode:
            setup.setup_vscode()

        if not args.skip_pycharm:
            setup.setup_pycharm()

    sys.exit(0 if success else 1)
setup

TB Language Setup Utility - File association (.tbx files) - Icon registration - Desktop integration

TBxSetup

Setup utility for TB Language file associations and icons

Source code in toolboxv2/utils/tbx/setup.py
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
class TBxSetup:
    """Setup utility for TB Language file associations and icons"""

    def __init__(self):
        self.system = platform.system()
        self.tb_root = self.get_tb_root()
        self.icon_path = Path(os.getenv("FAVI", ".ico"))
        self.executable = self.get_executable()

    def get_tb_root(self) -> Path:
        """Get toolbox root directory"""
        try:
            from toolboxv2 import tb_root_dir
            return Path(tb_root_dir)
        except ImportError:
            return Path(__file__).parent.parent

    def get_executable(self) -> Path:
        """Get TB executable path"""
        if self.system == "Windows":
            exe = self.tb_root / "bin" / "tb.exe"
        else:
            exe = self.tb_root / "bin" / "tb"

        if not exe.exists():
            # Try target/release
            if self.system == "Windows":
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
            else:
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

        return exe

    def setup_all(self):
        """Run complete setup"""
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║         TB Language - System Integration Setup                 ║")
        print("╚════════════════════════════════════════════════════════════════╝")
        print()

        # Check prerequisites
        if not self.executable.exists():
            print("❌ TB executable not found!")
            print(f"   Expected at: {self.executable}")
            print("   Run 'tb x build' first!")
            return False

        print(f"✓ TB executable found: {self.executable}")
        print()

        # Setup icon
        if not self.setup_icon():
            print("⚠️  Icon setup failed (continuing anyway)")

        # Setup file association
        if self.system == "Windows":
            success = self.setup_windows()
        elif self.system == "Linux":
            success = self.setup_linux()
        elif self.system == "Darwin":
            success = self.setup_macos()
        else:
            print(f"❌ Unsupported system: {self.system}")
            return False

        if success:
            print()
            print("╔════════════════════════════════════════════════════════════════╗")
            print("║                    ✓ Setup Complete!                           ║")
            print("╠════════════════════════════════════════════════════════════════╣")
            print("║  .tbx files are now associated with TB Language                ║")
            print("║  Double-click any .tbx file to run it!                         ║")
            print("╚════════════════════════════════════════════════════════════════╝")

        return success

    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False

    def setup_windows(self) -> bool:
        """Setup file association on Windows"""
        print("🪟 Setting up Windows file association...")

        try:
            import winreg

            # Create .tbx extension key
            print("   Creating registry entries...")

            # HKEY_CURRENT_USER\Software\Classes\.tbx
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
                print("   ✓ Registered .tbx extension")

            # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

                # Set icon
                if self.icon_path.exists():
                    icon_key = winreg.CreateKey(key, "DefaultIcon")
                    winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                    print(f"   ✓ Set icon: {self.icon_path}")

                # Set open command
                command_key = winreg.CreateKey(key, r"shell\open\command")
                cmd = f'"{self.executable}" run "%1"'
                winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
                print(f"   ✓ Set open command: {cmd}")

                # Add "Run in Terminal" context menu
                terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
                terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
                winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
                winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
                print(f"   ✓ Added 'Run in Terminal' context menu")

                # Add "Edit" context menu
                edit_key = winreg.CreateKey(key, r"shell\edit\command")
                winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
                winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
                print(f"   ✓ Added 'Edit' context menu")

            # Refresh shell
            print("   Refreshing Explorer...")
            try:
                import ctypes
                ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
            except:
                print("   ⚠️  Could not refresh Explorer (restart may be needed)")

            print("   ✓ Windows setup complete!")
            return True

        except ImportError:
            print("   ❌ winreg module not available")
            return False
        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def uninstall(self):
        """Remove file associations"""
        print("🗑️  Uninstalling file associations...")

        if self.system == "Windows":
            try:
                import winreg
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
                print("   ✓ Windows registry cleaned")
            except:
                print("   ⚠️  Could not clean registry")

        elif self.system == "Linux":
            desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
            mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

            if desktop_file.exists():
                desktop_file.unlink()
                print("   ✓ Removed desktop entry")

            if mime_file.exists():
                mime_file.unlink()
                print("   ✓ Removed MIME type")

        elif self.system == "Darwin":
            app_dir = self.tb_root / "TB Language.app"
            if app_dir.exists():
                shutil.rmtree(app_dir)
                print("   ✓ Removed app bundle")

        print("   ✓ Uninstall complete!")
get_executable()

Get TB executable path

Source code in toolboxv2/utils/tbx/setup.py
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
def get_executable(self) -> Path:
    """Get TB executable path"""
    if self.system == "Windows":
        exe = self.tb_root / "bin" / "tb.exe"
    else:
        exe = self.tb_root / "bin" / "tb"

    if not exe.exists():
        # Try target/release
        if self.system == "Windows":
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
        else:
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

    return exe
get_tb_root()

Get toolbox root directory

Source code in toolboxv2/utils/tbx/setup.py
26
27
28
29
30
31
32
def get_tb_root(self) -> Path:
    """Get toolbox root directory"""
    try:
        from toolboxv2 import tb_root_dir
        return Path(tb_root_dir)
    except ImportError:
        return Path(__file__).parent.parent
setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/setup.py
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def setup_all(self):
    """Run complete setup"""
    print("╔════════════════════════════════════════════════════════════════╗")
    print("║         TB Language - System Integration Setup                 ║")
    print("╚════════════════════════════════════════════════════════════════╝")
    print()

    # Check prerequisites
    if not self.executable.exists():
        print("❌ TB executable not found!")
        print(f"   Expected at: {self.executable}")
        print("   Run 'tb x build' first!")
        return False

    print(f"✓ TB executable found: {self.executable}")
    print()

    # Setup icon
    if not self.setup_icon():
        print("⚠️  Icon setup failed (continuing anyway)")

    # Setup file association
    if self.system == "Windows":
        success = self.setup_windows()
    elif self.system == "Linux":
        success = self.setup_linux()
    elif self.system == "Darwin":
        success = self.setup_macos()
    else:
        print(f"❌ Unsupported system: {self.system}")
        return False

    if success:
        print()
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║                    ✓ Setup Complete!                           ║")
        print("╠════════════════════════════════════════════════════════════════╣")
        print("║  .tbx files are now associated with TB Language                ║")
        print("║  Double-click any .tbx file to run it!                         ║")
        print("╚════════════════════════════════════════════════════════════════╝")

    return success
setup_icon()

Setup icon file

Source code in toolboxv2/utils/tbx/setup.py
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False
setup_linux()

Setup file association on Linux

Source code in toolboxv2/utils/tbx/setup.py
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False
setup_macos()

Setup file association on macOS

Source code in toolboxv2/utils/tbx/setup.py
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False
setup_windows()

Setup file association on Windows

Source code in toolboxv2/utils/tbx/setup.py
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def setup_windows(self) -> bool:
    """Setup file association on Windows"""
    print("🪟 Setting up Windows file association...")

    try:
        import winreg

        # Create .tbx extension key
        print("   Creating registry entries...")

        # HKEY_CURRENT_USER\Software\Classes\.tbx
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
            print("   ✓ Registered .tbx extension")

        # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

            # Set icon
            if self.icon_path.exists():
                icon_key = winreg.CreateKey(key, "DefaultIcon")
                winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                print(f"   ✓ Set icon: {self.icon_path}")

            # Set open command
            command_key = winreg.CreateKey(key, r"shell\open\command")
            cmd = f'"{self.executable}" run "%1"'
            winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
            print(f"   ✓ Set open command: {cmd}")

            # Add "Run in Terminal" context menu
            terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
            terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
            winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
            winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
            print(f"   ✓ Added 'Run in Terminal' context menu")

            # Add "Edit" context menu
            edit_key = winreg.CreateKey(key, r"shell\edit\command")
            winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
            winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
            print(f"   ✓ Added 'Edit' context menu")

        # Refresh shell
        print("   Refreshing Explorer...")
        try:
            import ctypes
            ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
        except:
            print("   ⚠️  Could not refresh Explorer (restart may be needed)")

        print("   ✓ Windows setup complete!")
        return True

    except ImportError:
        print("   ❌ winreg module not available")
        return False
    except Exception as e:
        print(f"   ❌ Error: {e}")
        return False
uninstall()

Remove file associations

Source code in toolboxv2/utils/tbx/setup.py
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
def uninstall(self):
    """Remove file associations"""
    print("🗑️  Uninstalling file associations...")

    if self.system == "Windows":
        try:
            import winreg
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
            print("   ✓ Windows registry cleaned")
        except:
            print("   ⚠️  Could not clean registry")

    elif self.system == "Linux":
        desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
        mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

        if desktop_file.exists():
            desktop_file.unlink()
            print("   ✓ Removed desktop entry")

        if mime_file.exists():
            mime_file.unlink()
            print("   ✓ Removed MIME type")

    elif self.system == "Darwin":
        app_dir = self.tb_root / "TB Language.app"
        if app_dir.exists():
            shutil.rmtree(app_dir)
            print("   ✓ Removed app bundle")

    print("   ✓ Uninstall complete!")
main()

Main entry point

Source code in toolboxv2/utils/tbx/setup.py
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language System Integration Setup"
    )
    parser.add_argument('action', choices=['install', 'uninstall'],
                        help='Action to perform')

    args = parser.parse_args()

    setup = TBxSetup()

    if args.action == 'install':
        success = setup.setup_all()
        sys.exit(0 if success else 1)
    elif args.action == 'uninstall':
        setup.uninstall()
        sys.exit(0)
test
test_tb_lang

TB Language Comprehensive Test Suite Tests all features of the TB language implementation.

Usage

python test_tb_lang.py python test_tb_lang.py --verbose python test_tb_lang.py --filter "test_arithmetic"

assert_contains(code, substring, mode='jit')

Assert that output contains substring.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
352
353
354
355
356
357
358
359
360
361
362
def assert_contains(code: str, substring: str, mode: str = "jit"):
    """Assert that output contains substring."""
    success, stdout, stderr = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    if substring not in stdout:
        raise AssertionError(
            f"Output does not contain '{substring}':\n{stdout}"
        )
assert_output(code, expected, mode='jit')

Assert that TB code produces expected output.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
def assert_output(code: str, expected: str, mode: str = "jit"):
    """Assert that TB code produces expected output."""

    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
        print(f"stdout: {stdout}")
    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()

    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\n"
            f"Expected: {repr(expected)}\n"
            f"Got:      {repr(actual)}"
        )
assert_success(code, mode='jit')

Assert that TB code runs without error.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
338
339
340
341
342
343
344
345
346
347
348
349
def assert_success(code: str, mode: str = "jit"):
    """Assert that TB code runs without error."""
    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
    if VERBOSE:
        print(f"stdout: {stdout}")
        print(f"stderr: {stderr}")

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")
find_tb_binary()

Find TB binary, trying multiple paths with system-specific extensions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def find_tb_binary() -> str:
    """Find TB binary, trying multiple paths with system-specific extensions."""
    from toolboxv2 import tb_root_dir
    # Path to TB binary (relative or absolute)
    TB_BINARY_PATH = str(tb_root_dir / "bin" / "tbx")

    # Alternative paths to try if main path doesn't exist
    ALTERNATIVE_PATHS = [
        str(tb_root_dir / "tb-exc" / "target" / "debug" / "tbx"),
        str(tb_root_dir / "tb-exc" / "target" / "release" / "tbx"),
        #"tbx",  # System PATH
    ]

    # Add system-specific extension
    def add_extension(path: str) -> str:
        if system() == "Windows" and not path.endswith(".exe"):
            return f"{path}.exe"
        return path

    # Prepare paths with proper extensions
    paths_to_try = [add_extension(TB_BINARY_PATH)]
    for alt_path in ALTERNATIVE_PATHS:
        paths_to_try.append(add_extension(alt_path))

    for path in paths_to_try:
        # Check if file exists directly
        if os.path.exists(path):
            return path

        # Use shutil.which for system PATH lookup (cross-platform)
        if shutil.which(path):
            return path

    print(f"{Colors.RED}✗ TB binary not found!{Colors.RESET}")
    print(f"{Colors.YELLOW}Tried paths:{Colors.RESET}")
    for path in paths_to_try:
        print(f"  • {path}")
    print(f"\n{Colors.CYAN}Build the binary with:{Colors.RESET}")
    print(f"  tb run build")
    return ""
run_tb(code, mode='jit', timeout=10)

Run TB code and return (success, stdout, stderr).

Parameters:

Name Type Description Default
code str

TB source code

required
mode str

Execution mode (jit, compiled, streaming)

'jit'
timeout int

Timeout in seconds

10

Returns:

Type Description
Tuple[bool, str, str]

(success, stdout, stderr)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def run_tb(code: str, mode: str = "jit", timeout: int = 10) -> Tuple[bool, str, str]:
    """
    Run TB code and return (success, stdout, stderr).

    Args:
        code: TB source code
        mode: Execution mode (jit, compiled, streaming)
        timeout: Timeout in seconds

    Returns:
        (success, stdout, stderr)
    """
    if mode == "compiled":
        with tempfile.NamedTemporaryFile(suffix='', delete=False) as f:
            output_path = f.name

        try:
            start = time.perf_counter()
            success, stderr = run_tb_compile(code, output_path)
            duration = time.perf_counter() - start
            print(f" -- Compile time ({duration:.3f}s)")
            if not success:
                raise AssertionError(f"Compilation failed:\n{stderr}")

            # Check binary exists
            if not os.path.exists(output_path):
                raise AssertionError("Compiled binary not found")

            # Check binary is executable
            if not os.access(output_path, os.X_OK):
                os.chmod(output_path, 0o755)

            start = time.perf_counter()
            # Run compiled binary
            result = subprocess.run([output_path], capture_output=True, text=True, timeout=timeout//2,
                                    encoding=sys.stdout.encoding or 'utf-8')
            duration = time.perf_counter() - start
            print(f" -- Exec time ({duration:.3f}s)")

            if result.returncode != 0:
                raise AssertionError(f"Compiled binary failed: {result.stderr}")

            return success, result.stdout, result.stderr
        finally:
            if os.path.exists(output_path):
                os.remove(output_path)

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        result = subprocess.run(
            [TB_BINARY, "run", temp_file, "--mode", mode],
            capture_output=True,
            text=True,
            timeout=timeout,
            encoding=sys.stdout.encoding or 'utf-8',
            errors='replace'
        )

        success = result.returncode == 0
        return success, result.stdout, result.stderr

    except subprocess.TimeoutExpired:
        return False, "", f"Timeout after {timeout}s"

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass
run_tb_compile(code, output_path, target=None)

Compile TB code to binary.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
def run_tb_compile(code: str, output_path: str, target: str = None) -> Tuple[bool, str]:
    """Compile TB code to binary."""
    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        cmd = [TB_BINARY, "compile", temp_file, output_path]
        if target:
            cmd.extend(["--target", target])

        result = subprocess.run(cmd, capture_output=not VERBOSE, text=True, timeout=60, encoding=sys.stdout.encoding or 'utf-8')

        success = result.returncode == 0
        return success, result.stderr

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass
test(name, category='General')

Decorator for test functions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
def test(name: str, category: str = "General"):
    """Decorator for test functions."""

    def decorator(func):
        def wrapper():
            # Check filter
            if FILTER and FILTER.lower() not in name.lower():
                return

            # Print test header
            if suite.current_category != category:
                print(f"\n{Colors.BOLD}{Colors.CYAN}[{category}]{Colors.RESET}")
                suite.current_category = category

            print(f"  {Colors.GRAY}Testing:{Colors.RESET} {name}", end=" ", flush=True)

            start = time.perf_counter()
            try:
                func()
                duration = time.perf_counter() - start

                print(f" -- {Colors.GREEN}{Colors.RESET} ({duration:.3f}s)")

                suite.add_result(TestResult(
                    name=name,
                    passed=True,
                    duration_ms=duration
                ))

            except AssertionError as e:
                duration = time.perf_counter() - start

                print(f"{Colors.RED}{Colors.RESET} ({duration:.0f}s)")
                if VERBOSE:
                    print(f"    {Colors.RED}Error: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=str(e)
                ))

            except Exception as e:
                import traceback
                print(traceback.format_exc())
                duration = (time.perf_counter() - start) * 1000

                print(f"{Colors.RED}✗ (Exception){Colors.RESET}")
                if VERBOSE:
                    print(f"    {Colors.RED}{type(e).__name__}: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=f"{type(e).__name__}: {str(e)}"
                ))

        return wrapper

    return decorator
test_compiled_arithmetic()

Test basic arithmetic in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
@test("Compiled - Basic Arithmetic", "Compiled Modes")
def test_compiled_arithmetic():
    """Test basic arithmetic in compiled mode"""
    code = '''
let x = 10
let y = 20
let sum = x + y
let product = x * y
echo sum
echo product
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    assert "30" in stdout
    assert "200" in stdout
test_compiled_functions()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
@test("Compiled - Functions", "Compiled Modes")
def test_compiled_functions():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="compiled")
test_compiled_functions_no_compiled_()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
@test("Compiled - Functions", "Compiled Modes NO Compiled")
def test_compiled_functions_no_compiled_():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="jit")
test_compiled_loops()

Test loops in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
@test("Compiled - Loops", "Compiled Modes")
def test_compiled_loops():
    """Test loops in compiled mode"""
    code = '''
let sum = 0
for i in [1, 2, 3, 4, 5] {
    sum = sum + i
}
echo sum
'''
    assert_output(code, "15", mode="compiled")
test_compiled_parallel()

Test parallel execution in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
@test("Compiled - Parallel Execution", "Compiled Modes")
def test_compiled_parallel():
    """Test parallel execution in compiled mode"""
    code = '''
@config {
    runtime_mode: "parallel"
}

let results = parallel {
    10 + 5,
    20 * 2,
    30 - 10
}
for result in results {
    echo result
}
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    # Results may be in any order due to parallelism
    assert "15" in stdout, f"result {stdout}"
    assert "40" in stdout, f"result {stdout}"
    assert "20" in stdout, f"result {stdout}"
test_compiled_parallel_with_imports()

Test parallel execution with imported functions

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
@test("Compiled - Parallel with Imports", "Compiled Modes")
def test_compiled_parallel_with_imports():
    """Test parallel execution with imported functions"""
    lib_code = '''
fn compute_square(x: int) {
    x * x
}

fn compute_cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{lib_path}"
}}

let results = parallel {{
    compute_square(5),
    compute_cube(3),
    compute_square(10)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "25" in stdout
        assert "27" in stdout
        assert "100" in stdout
    finally:
        os.unlink(lib_path)
test_go_basic_execution()

Test that Go code can be executed.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
@test("Go Language Bridge - Basic Execution", "Dependencies - Go Bridge")
def test_go_basic_execution():
    """Test that Go code can be executed."""
    code = '''
let result = go("""
import "fmt"
fmt.Println("Hello from Go!")
""")
'''
    assert_success(code)
test_go_bridge_compiled_mode()

Test that the Go language bridge works in compiled mode.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
@test("Go Bridge - Compiled Mode", "Dependencies - Go Bridge")
def test_go_bridge_compiled_mode():
    """Test that the Go language bridge works in compiled mode."""
    code = '''
@config {
    mode: "compiled"
}

let result = go("""
import "fmt"
fmt.Println("Hello from Go in compiled mode!")
""")

echo "Go execution finished."
'''
    assert_contains(code, "Go execution finished.", mode="compiled")
test_go_import_json()

Test marshalling data to JSON using Go's 'encoding/json' package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
@test("Go Import - JSON Marshalling", "Dependencies - Go Bridge")
def test_go_import_json():
    """Test marshalling data to JSON using Go's 'encoding/json' package."""
    code = '''
# This test assumes TB lists/dicts are converted to Go slices/maps
let user_data = [
    { "name": "Alice", "age": 30 },
    { "name": "Bob", "age": 25 }
]
let result = go("""
import (
    "fmt"
    "encoding/json"
)
// user_data is available from TB, assume it's a []map[string]interface{}
// Marshal returns the JSON encoding of user_data.
jsonData, err := json.Marshal(user_data)
if err != nil {
    fmt.Println("Error:", err)
} else {
    fmt.Println(string(jsonData))
}
""")
'''
    assert_success(code)
test_go_import_math()

Test using the Go 'math' standard library package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
@test("Go Import - Math", "Dependencies - Go Bridge")
def test_go_import_math():
    """Test using the Go 'math' standard library package."""
    code = '''
let result = go("""
import (
    "fmt"
    "math"
)
// Sqrt returns the square root of 64
fmt.Println(math.Sqrt(64))
""")
'''
    assert_success(code)
test_go_import_strings()

Test using the Go 'strings' package with a TB variable.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
@test("Go Import - Strings", "Dependencies - Go Bridge")
def test_go_import_strings():
    """Test using the Go 'strings' package with a TB variable."""
    code = '''
let my_string = "Hello, Go!"
let result = go("""
import (
    "fmt"
    "strings"
)
// my_string is available from TB context
// ToUpper returns a copy of the string with all letters mapped to their upper case.
fmt.Println(strings.ToUpper(my_string))
""")
'''
    assert_success(code)
test_go_multilang_python_to_go()

Test data flow from a Python block to a Go block.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
@test("Go Multi-Language - Python to Go", "Dependencies - Go Bridge")
def test_go_multilang_python_to_go():
    """Test data flow from a Python block to a Go block."""
    code = '''
# Python creates a string
let data_from_py = python("""
print("Data generated by Python")
""")

# Go processes the string from the TB variable
let go_result = go("""
import "fmt"
// data_from_py is available from TB context
// Printf formats according to a format specifier and writes to standard output.
fmt.Printf("Go received: '%s'\\n", data_from_py)
""")
'''
    assert_success(code)
test_import_cache_invalidation()

Test that cache is invalidated when source changes

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
@test("Compiled - Cache Invalidation", "Import Compiled Modes")
def test_import_cache_invalidation():
    """Test that cache is invalidated when source changes"""
    print("→ Test: Cache Invalidation")

    lib_path = 'test_cache_lib.tbx'

    # Write initial version
    with open(lib_path, 'w') as f:
        f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    42
}
''')

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo get_value()
'''

        # First run
        success1, stdout1, _ = run_tb(main_code)
        assert success1
        assert "42" in stdout1

        # Modify library
        time.sleep(0.1)  # Ensure timestamp difference
        with open(lib_path, 'w') as f:
            f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    100
}
''')

        # Second run should use updated version
        success2, stdout2, _ = run_tb(main_code)
        assert success2
        assert "100" in stdout2, f"Cache not invalidated! Got: {stdout2}"

        print("  ✓ Cache invalidation works")

    finally:
        os.unlink(lib_path)
test_import_compiled_caching()

Test that compiled imports are cached

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
@test("Compiled - Import Caching", "Import Compiled Modes")
def test_import_compiled_caching():
    """Test that compiled imports are cached"""
    print("→ Test: Compiled Import Caching")

    # Create library with compiled mode
    lib_code = '''
@config {
    mode: "compiled"
}

fn factorial(n: int) {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = factorial(5)
echo $result
'''

        # First run - should compile
        start_time = time.perf_counter()
        success1, stdout1, stderr1 = run_tb(main_code)
        first_run_time = time.perf_counter() - start_time

        assert success1, f"First execution failed: {stderr1}"
        assert "120" in stdout1, f"Expected 120, got: {stdout1}"

        # Second run - should use cache
        start_time = time.perf_counter()
        success2, stdout2, stderr2 = run_tb(main_code)
        second_run_time = time.perf_counter() - start_time

        assert success2, f"Second execution failed: {stderr2}"
        assert "120" in stdout2, f"Expected 120, got: {stdout2}"

        # Second run should be significantly faster (using cache)
        assert second_run_time <= first_run_time, \
            f"Cache not used? First: {first_run_time:.2f}s, Second: {second_run_time:.2f}s"

        print(f"  ✓ Caching works (first: {first_run_time:.2f}s, second: {second_run_time:.2f}s)")

    finally:
        os.unlink(lib_path)
test_import_compiled_mode()

Test that imports work in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
@test("Import - Compiled Mode", "Import System")
def test_import_compiled_mode():
    """Test that imports work in compiled mode"""
    lib_code = '''
@config {
    mode: "compiled"
    target: "library"
}

fn double(x: int) {
    x * 2
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = double(21)
echo result
'''
        assert_output(main_code, "42", mode="compiled")
    finally:
        os.unlink(lib_path)
test_import_dependency_chain()

Test imports that depend on other imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
@test("Compiled - Import Dependency Chain", "Import Compiled Modes")
def test_import_dependency_chain():
    """Test imports that depend on other imports"""
    print("→ Test: Import Dependency Chain")

    # Base library
    base_lib = '''
fn double(x: int) {
    x * 2
}
'''

    # Mid library (uses base)
    mid_lib = '''
@imports {
    "base.tbx"
}

fn quadruple(x: int) {
    double(double(x))
}
'''

    # Create base.tbx
    with open('base.tbx', 'w') as f:
        f.write(base_lib)

    # Create mid.tbx
    with open('mid.tbx', 'w') as f:
        f.write(mid_lib)

    try:
        main_code = '''
@imports {
    "mid.tbx"
}

echo quadruple(5)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "20" in stdout, f"Expected 20, got: {stdout}"

        print("  ✓ Transitive imports work")

    finally:
        os.unlink('base.tbx')
        os.unlink('mid.tbx')
test_import_jit_vs_compiled()

Test that JIT imports don't get compiled

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
@test("Compiled - JIT vs Compiled Imports", "Import Compiled Modes")
def test_import_jit_vs_compiled():
    """Test that JIT imports don't get compiled"""
    print("→ Test: JIT vs Compiled Imports")

    # JIT library
    jit_lib = '''
@config {
    mode: "jit"
}

fn square(x: int) {
    x * x
}
'''

    # Compiled library
    compiled_lib = '''
@config {
    mode: "compiled"
}

fn cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as jit_file, \
         tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as compiled_file:

        jit_file.write(jit_lib)
        compiled_file.write(compiled_lib)
        jit_path = os.path.basename(jit_file.name)
        compiled_path = os.path.basename(compiled_file.name)

    try:
        main_code = f'''
@imports {{
    "{jit_path}"
    "{compiled_path}"
}}

echo square(4)
echo cube(3)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "16" in stdout, f"Expected 16 from square(4), got: {stdout}"
        assert "27" in stdout, f"Expected 27 from cube(3), got: {stdout}"

        print("  ✓ Mixed JIT/Compiled imports work")

    finally:
        os.unlink(jit_path)
        os.unlink(compiled_path)
test_import_missing_file()

Test error handling for missing import files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
@test("Import - Missing File Error", "Import Errors")
def test_import_missing_file():
    """Test error handling for missing import files"""
    code = '''
@imports {
    "nonexistent_file.tbx"
}

echo "This should not run"
'''
    success, stdout, stderr = run_tb(code)
    assert not success, "Should fail for missing import file"
    assert "not found" in stderr.lower() or "import" in stderr.lower()
test_import_multiple_files()

Test importing multiple .tbx files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
@test("Import - Multiple Files", "Import System")
def test_import_multiple_files():
    """Test importing multiple .tbx files"""
    # Create math library
    math_lib = '''
fn add(x, y) {
    x + y
}
'''

    # Create string library
    string_lib = '''
fn greet(name: string) {
    echo "Hello,"name "!"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as string_file:

        math_file.write(math_lib)
        string_file.write(string_lib)
        math_path = os.path.basename(math_file.name)
        string_path = os.path.basename(string_file.name)

    try:
        main_code = f'''
@imports {{
    "{math_path}"
    "{string_path}"
}}

let sum = add(3, 7)
echo sum
greet("World")
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "10" in stdout, f"Expected '10' in output, got: {stdout}"
        assert "Hello, World !" in stdout, f"Expected Hello, World! in output, got: {stdout}"
    finally:
        os.unlink(math_path)
        os.unlink(string_path)
test_import_no_circular()

Test that circular imports are handled (imports are not recursive)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
@test("Import - Circular Import Protection", "Import Errors")
def test_import_no_circular():
    """Test that circular imports are handled (imports are not recursive)"""
    # Create file that tries to import itself
    lib_code = '''
fn test_func() {
    echo "Hello"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # This should work because imports are not recursive
        main_code = f'''
@imports {{
    "{lib_path}"
}}

test_func()
'''
        assert_success(main_code)
    finally:
        os.unlink(lib_path)
test_import_relative_paths()

Test importing with relative directory paths

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
@test("Import - Relative Paths", "Import System")
def test_import_relative_paths():
    """Test importing with relative directory paths"""
    # Create subdirectory
    lib_dir = tempfile.mkdtemp(dir='..')
    lib_dir_name = os.path.basename(lib_dir)

    try:
        # Create library in subdirectory
        lib_code = '''
fn multiply(x: int, y: int) {
    x * y
}
'''
        lib_path = os.path.join(lib_dir, 'math.tbx')
        with open(lib_path, 'w') as f:
            f.write(lib_code)

        main_code = f'''
@imports {{
    "{lib_dir_name}/math.tbx"
}}

let result = multiply(6, 7)
echo result
'''
        assert_output(main_code, "42")
    finally:
        shutil.rmtree(lib_dir, ignore_errors=True)
test_import_single_file()

Test importing a single .tbx file

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
@test("Import - Single File", "Import System")
def test_import_single_file():
    """Test importing a single .tbx file"""
    # Create library file
    lib_code = '''
fn double(x: int) {
    x * 2
}

fn triple(x: int) {
    x * 3
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # Create main file that imports library
        main_code = f'''
@imports {{
    "{lib_path}"
}}

let result = double(5)
echo result
'''
        assert_output(main_code, "10")
    finally:
        os.unlink(lib_path)
test_import_with_variables()

Test importing file with global variables

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
@test("Import - With Variables", "Import System")
def test_import_with_variables():
    """Test importing file with global variables"""
    lib_code = '''
let PI = 3.14159
let E = 2.71828

fn circle_area(radius: float) {
    PI * radius * radius
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo PI
let area = circle_area(2.0)
echo area
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "3.14159" in stdout or "3.14" in stdout
    finally:
        os.unlink(lib_path)
test_mixed_full_stack()

Comprehensive test with all features combined

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
@test("Mixed - Full Stack Test", "Mixed Features")
def test_mixed_full_stack():
    """Comprehensive test with all features combined"""
    helpers_lib = '''
fn compute(x: int) {
    x * 2
}

fn parallel_sum(numbers: list) {
    let sum = 0
    for n in numbers {
        sum = sum + n
    }
    sum
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(helpers_lib)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "jit"
}}

@shared {{
    total: 0
}}

@imports {{
    "{lib_path}"
}}

let value = compute(21)
total = parallel_sum([1, 2, 3, 4, 5])
echo value
echo total
'''
        success, stdout, stderr = run_tb(main_code, mode="jit")
        assert success, f"Execution failed: {stderr}"
        assert "42" in stdout
        assert "15" in stdout
    finally:
        os.unlink(lib_path)
test_mixed_import_async_language()

Test imports with async and language bridges

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
@test("Mixed - Import  + Language Bridge", "Mixed Features")
def test_mixed_import_async_language():
    """Test imports with async and language bridges"""
    lib_code = '''
fn get_data() {

    python("import sys; print(sys.version.split()[0])")

}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@imports {{
    "{lib_path}"
}}

let version = get_data()
echo "Python version detected"
'''
        assert_contains(main_code, "Python version detected", mode="jit")
    finally:
        os.unlink(lib_path)
test_mixed_multiple_imports_parallel()

Test multiple imports with parallel execution

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
@test("Mixed - Multiple Imports + Parallel", "Mixed Features")
def test_mixed_multiple_imports_parallel():
    """Test multiple imports with parallel execution"""
    math_lib = '''
fn factorial(n: int) -> int {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    utils_lib = '''
fn power(base: int, exp: int) -> int {
    if exp == 0 {
        1
    } else {
        base * power(base, exp - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as utils_file:

        math_file.write(math_lib)
        utils_file.write(utils_lib)
        math_path = os.path.basename(math_file.name)
        utils_path = os.path.basename(utils_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{math_path}"
    "{utils_path}"
}}

let results = parallel {{
    factorial(5),
    power(2, 10),
    factorial(6)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "120" in stdout  # 5!
        assert "1024" in stdout  # 2^10
        assert "720" in stdout  # 6!
    finally:
        os.unlink(math_path)
        os.unlink(utils_path)
test_mixed_shared_imports()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
@test("Mixed - Shared Variables + Imports", "Mixed Features")
def test_mixed_shared_imports():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {
    counter + 1
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)
test_mixed_shared_imports_mixed_lang()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
@test("Mixed - Shared Variables + Imports + Python Bridge", "Mixed Features")
def test_mixed_shared_imports_mixed_lang():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {python("print(counter + 1, end='')")}

'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)
test_type_annotations()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("""x = "Alice"
x""")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")
test_type_annotations_()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations_():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("19.99")
let name: string = python("Alice")
let active: bool = python("True")
let scores: list<int> = python("[85, 92, 78]")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")
test_type_annotations_auto()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
@test("Type Annotations - Auto Type Inference", "Type Annotations")
def test_type_annotations_auto():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo type_of(age)           // Age: 42
echo type_of(price)       // Price: 19.99
echo type_of(name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist")
test_type_annotations_auto_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
@test("Type Annotations - Auto Type Inference  - Compiled", "Type Annotations - Compiled")
def test_type_annotations_auto_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list<int> = python("print([85, 92, 78])")

echo type_of($age)           // Age: 42
echo type_of($price)       // Price: 19.99
echo type_of($name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist<int>", mode="compiled")
test_type_annotations_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
@test("Type Annotations - Basic Type - Compiled", "Type Annotations - Compiled")
def test_type_annotations_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true", mode="compiled")
test_tb_lang2

TB Language Comprehensive Test Suite Tests all features of the TB language implementation.

Usage

python test_tb_lang.py python test_tb_lang.py --verbose python test_tb_lang.py --filter "test_arithmetic" python test_tb_lang.py --mode jit python test_tb_lang.py --mode compiled python test_tb_lang.py --skip-slow

TestSuite
Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
class TestSuite:
    def __init__(self):
        self.results: List[TestResult] = []
        self.current_category = ""
        self.failed_filter = None
        self.failed_tests_cache = self.load_failed_tests()

    def load_failed_tests(self) -> set:
        """Load previously failed test names from cache."""
        cache_file = Path(__file__).parent / ".failed_tests.json"
        if cache_file.exists():
            try:
                with open(cache_file, 'r') as f:
                    data = json.load(f)
                    return set(data.get('failed_tests', []))
            except:
                pass
        return set()

    def save_failed_tests(self):
        """Save failed test names to cache."""
        cache_file = Path(__file__).parent / ".failed_tests.json"
        failed_names = [r.name for r in self.results if not r.passed]
        with open(cache_file, 'w') as f:
            json.dump({'failed_tests': failed_names}, f, indent=2)

    def should_run_test(self, test_name: str) -> bool:
        """Check if test should run based on FAILED_ONLY flag."""
        if not FAILED_ONLY:
            return True
        return test_name in self.failed_tests_cache

    def add_result(self, result: TestResult):
        self.results.append(result)

    def print_summary(self):
        total = len(self.results)
        passed = sum(1 for r in self.results if r.passed)
        failed = total - passed
        total_time = sum(r.duration_ms for r in self.results)

        print("\n" + "=" * 80)
        print(f"{Colors.BOLD}TEST SUMMARY{Colors.RESET}")
        print("=" * 80)

        if failed == 0:
            print(f"{Colors.GREEN}OK - All {total} tests passed!{Colors.RESET}")
        else:
            print(f"{Colors.RED}FAILED - {failed} of {total} tests failed{Colors.RESET}")
            print(f"{Colors.GREEN}OK - {passed} passed{Colors.RESET}")

        print(f"\n{Colors.CYAN}Total time: {total_time:.2f}ms{Colors.RESET}")

        # Performance statistics
        jit_results = [r for r in self.results if r.mode == "jit" and r.passed]
        compiled_results = [r for r in self.results if r.mode == "compiled" and r.passed]

        if jit_results:
            avg_jit = sum(r.duration_ms for r in jit_results) / len(jit_results)
            print(f"{Colors.BLUE}JIT avg time: {avg_jit:.2f}ms{Colors.RESET}")

        if compiled_results:
            avg_compiled = sum(r.duration_ms for r in compiled_results) / len(compiled_results)
            avg_compile = sum(r.compile_time_ms for r in compiled_results if r.compile_time_ms) / len(compiled_results)
            avg_exec = sum(r.exec_time_ms for r in compiled_results if r.exec_time_ms) / len(compiled_results)
            print(
                f"{Colors.BLUE}Compiled avg time: {avg_compiled:.2f}ms (compile: {avg_compile:.2f}ms, exec: {avg_exec:.2f}ms){Colors.RESET}")

        if failed > 0:
            print(f"\n{Colors.RED}Failed tests:{Colors.RESET}")
            for result in self.results:
                if not result.passed:
                    print(f"  - {result.name} ({result.mode})")
                    if result.error_message:
                        # Encode error message safely to avoid Unicode issues
                        try:
                            print(f"    {Colors.GRAY}{result.error_message}{Colors.RESET}")
                        except UnicodeEncodeError:
                            # Fallback: print without special characters
                            safe_msg = result.error_message.encode('ascii', 'replace').decode('ascii')
                            print(f"    {Colors.GRAY}{safe_msg}{Colors.RESET}")

        return failed == 0
load_failed_tests()

Load previously failed test names from cache.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
84
85
86
87
88
89
90
91
92
93
94
def load_failed_tests(self) -> set:
    """Load previously failed test names from cache."""
    cache_file = Path(__file__).parent / ".failed_tests.json"
    if cache_file.exists():
        try:
            with open(cache_file, 'r') as f:
                data = json.load(f)
                return set(data.get('failed_tests', []))
        except:
            pass
    return set()
save_failed_tests()

Save failed test names to cache.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
 96
 97
 98
 99
100
101
def save_failed_tests(self):
    """Save failed test names to cache."""
    cache_file = Path(__file__).parent / ".failed_tests.json"
    failed_names = [r.name for r in self.results if not r.passed]
    with open(cache_file, 'w') as f:
        json.dump({'failed_tests': failed_names}, f, indent=2)
should_run_test(test_name)

Check if test should run based on FAILED_ONLY flag.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
103
104
105
106
107
def should_run_test(self, test_name: str) -> bool:
    """Check if test should run based on FAILED_ONLY flag."""
    if not FAILED_ONLY:
        return True
    return test_name in self.failed_tests_cache
assert_contains(code, substring, mode='jit')

Assert that output contains substring.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
401
402
403
404
405
406
407
408
409
def assert_contains(code: str, substring: str, mode: str = "jit"):
    """Assert that output contains substring."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    if substring not in stdout:
        raise AssertionError(f"Output does not contain '{substring}':\n{stdout}")
assert_error(code, mode='jit')

Assert that code fails.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
412
413
414
415
416
417
def assert_error(code: str, mode: str = "jit"):
    """Assert that code fails."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if success:
        raise AssertionError(f"Expected failure but succeeded:\n{stdout}")
assert_output(code, expected, mode='jit')

Assert that TB code produces expected output.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
372
373
374
375
376
377
378
379
380
381
382
383
384
385
def assert_output(code: str, expected: str, mode: str = "jit"):
    """Assert that TB code produces expected output."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()

    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\nExpected: {repr(expected)}\nGot: {repr(actual)}"
        )
assert_output_with_tcp_server(code, expected, mode='jit', host='localhost', port=8085)

Run code while a temporary TCP server is alive. The server accepts a single connection, reads once, then closes.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
def assert_output_with_tcp_server(code: str, expected: str, mode: str = "jit",
                                  host: str = "localhost", port: int = 8085):
    """
    Run code while a temporary TCP server is alive.
    The server accepts a single connection, reads once, then closes.
    """
    received = []

    def _server():
        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
            s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            s.bind((host, port))
            s.listen(1)
            conn, addr = s.accept()
            with conn:
                data = conn.recv(4096)
                if data:
                    received.append(data)

    t = threading.Thread(target=_server, daemon=True)
    t.start()

    # run TB code
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()
    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\nExpected: {repr(expected)}\nGot: {repr(actual)}"
        )

    # optionally validate something was actually received
    if not received:
        raise AssertionError("TCP server received no data")
assert_success(code, mode='jit')

Assert that TB code runs without error.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
388
389
390
391
392
393
394
395
396
397
398
def assert_success(code: str, mode: str = "jit"):
    """Assert that TB code runs without error."""
    success, stdout, stderr, compile_time, exec_time = run_tb(code, mode)

    if VERBOSE:
        print(f"\n    stdout: {stdout}")
        if stderr:
            print(f"    stderr: {stderr}")

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")
escape_path_for_tb(path)

Escape path for TB string literals.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
3677
3678
3679
def escape_path_for_tb(path):
    """Escape path for TB string literals."""
    return path.replace('\\', '\\\\')
find_tb_binary()

Find TB binary in multiple locations.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
def find_tb_binary() -> str:
    """Find TB binary in multiple locations."""
    try:
        from toolboxv2 import tb_root_dir
        paths = [
            tb_root_dir / "tb-exc" /"src" / "target" / "release" / "tb",  # Prefer release for faster compilation
            tb_root_dir / "tb-exc" /"src" / "target" / "debug" / "tb",
            tb_root_dir / "bin" / "tb",
        ]
    except:
        paths = [
            Path("target/release/tb"),
            Path("target/debug/tb"),
            Path("tb"),
        ]

    paths = [os.environ.get("TB_EXE"), os.environ.get("TB_BINARY")]+paths
    # Add .exe for Windows
    if os.name == 'nt':
        paths = [Path(str(p) + ".exe") for p in paths if p is not None]

    for path in paths:
        if path is None:
            continue
        if shutil.which(str(path)) or os.path.exists(path):
            return str(path)

    print(f"{Colors.YELLOW}Tried paths:{Colors.RESET}")
    for path in paths:
        print(f"  • {path}")
    print(f"\n{Colors.CYAN}Build with: tb run build{Colors.RESET}")
load_failed_tests()

Load failed test names from file.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
222
223
224
225
226
227
228
229
230
def load_failed_tests():
    """Load failed test names from file."""
    try:
        if os.path.exists(FAILED_TESTS_FILE):
            with open(FAILED_TESTS_FILE, 'r', encoding='utf-8') as f:
                return set(line.strip() for line in f if line.strip())
    except Exception as e:
        print(f"{Colors.YELLOW}Warning: Could not load failed tests: {e}{Colors.RESET}")
    return set()
save_failed_tests(failed_names)

Save failed test names to file.

Source code in toolboxv2/utils/tbx/test/test_tb_lang2.py
213
214
215
216
217
218
219
220
def save_failed_tests(failed_names):
    """Save failed test names to file."""
    try:
        with open(FAILED_TESTS_FILE, 'w', encoding='utf-8') as f:
            for name in failed_names:
                f.write(f"{name}\n")
    except Exception as e:
        print(f"{Colors.YELLOW}Warning: Could not save failed tests: {e}{Colors.RESET}")

tbx_setup

TB Language Setup Utility - File association (.tbx files) - Icon registration - Desktop integration

TBxSetup

Setup utility for TB Language file associations and icons

Source code in toolboxv2/utils/tbx/setup.py
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
class TBxSetup:
    """Setup utility for TB Language file associations and icons"""

    def __init__(self):
        self.system = platform.system()
        self.tb_root = self.get_tb_root()
        self.icon_path = Path(os.getenv("FAVI", ".ico"))
        self.executable = self.get_executable()

    def get_tb_root(self) -> Path:
        """Get toolbox root directory"""
        try:
            from toolboxv2 import tb_root_dir
            return Path(tb_root_dir)
        except ImportError:
            return Path(__file__).parent.parent

    def get_executable(self) -> Path:
        """Get TB executable path"""
        if self.system == "Windows":
            exe = self.tb_root / "bin" / "tb.exe"
        else:
            exe = self.tb_root / "bin" / "tb"

        if not exe.exists():
            # Try target/release
            if self.system == "Windows":
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
            else:
                exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

        return exe

    def setup_all(self):
        """Run complete setup"""
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║         TB Language - System Integration Setup                 ║")
        print("╚════════════════════════════════════════════════════════════════╝")
        print()

        # Check prerequisites
        if not self.executable.exists():
            print("❌ TB executable not found!")
            print(f"   Expected at: {self.executable}")
            print("   Run 'tb x build' first!")
            return False

        print(f"✓ TB executable found: {self.executable}")
        print()

        # Setup icon
        if not self.setup_icon():
            print("⚠️  Icon setup failed (continuing anyway)")

        # Setup file association
        if self.system == "Windows":
            success = self.setup_windows()
        elif self.system == "Linux":
            success = self.setup_linux()
        elif self.system == "Darwin":
            success = self.setup_macos()
        else:
            print(f"❌ Unsupported system: {self.system}")
            return False

        if success:
            print()
            print("╔════════════════════════════════════════════════════════════════╗")
            print("║                    ✓ Setup Complete!                           ║")
            print("╠════════════════════════════════════════════════════════════════╣")
            print("║  .tbx files are now associated with TB Language                ║")
            print("║  Double-click any .tbx file to run it!                         ║")
            print("╚════════════════════════════════════════════════════════════════╝")

        return success

    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False

    def setup_windows(self) -> bool:
        """Setup file association on Windows"""
        print("🪟 Setting up Windows file association...")

        try:
            import winreg

            # Create .tbx extension key
            print("   Creating registry entries...")

            # HKEY_CURRENT_USER\Software\Classes\.tbx
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
                print("   ✓ Registered .tbx extension")

            # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
            with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
                winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

                # Set icon
                if self.icon_path.exists():
                    icon_key = winreg.CreateKey(key, "DefaultIcon")
                    winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                    print(f"   ✓ Set icon: {self.icon_path}")

                # Set open command
                command_key = winreg.CreateKey(key, r"shell\open\command")
                cmd = f'"{self.executable}" run "%1"'
                winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
                print(f"   ✓ Set open command: {cmd}")

                # Add "Run in Terminal" context menu
                terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
                terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
                winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
                winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
                print(f"   ✓ Added 'Run in Terminal' context menu")

                # Add "Edit" context menu
                edit_key = winreg.CreateKey(key, r"shell\edit\command")
                winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
                winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
                print(f"   ✓ Added 'Edit' context menu")

            # Refresh shell
            print("   Refreshing Explorer...")
            try:
                import ctypes
                ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
            except:
                print("   ⚠️  Could not refresh Explorer (restart may be needed)")

            print("   ✓ Windows setup complete!")
            return True

        except ImportError:
            print("   ❌ winreg module not available")
            return False
        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False

    def uninstall(self):
        """Remove file associations"""
        print("🗑️  Uninstalling file associations...")

        if self.system == "Windows":
            try:
                import winreg
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
                winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
                print("   ✓ Windows registry cleaned")
            except:
                print("   ⚠️  Could not clean registry")

        elif self.system == "Linux":
            desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
            mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

            if desktop_file.exists():
                desktop_file.unlink()
                print("   ✓ Removed desktop entry")

            if mime_file.exists():
                mime_file.unlink()
                print("   ✓ Removed MIME type")

        elif self.system == "Darwin":
            app_dir = self.tb_root / "TB Language.app"
            if app_dir.exists():
                shutil.rmtree(app_dir)
                print("   ✓ Removed app bundle")

        print("   ✓ Uninstall complete!")
get_executable()

Get TB executable path

Source code in toolboxv2/utils/tbx/setup.py
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
def get_executable(self) -> Path:
    """Get TB executable path"""
    if self.system == "Windows":
        exe = self.tb_root / "bin" / "tb.exe"
    else:
        exe = self.tb_root / "bin" / "tb"

    if not exe.exists():
        # Try target/release
        if self.system == "Windows":
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb.exe"
        else:
            exe = self.tb_root / "tb-exc" / "target" / "release" / "tb"

    return exe
get_tb_root()

Get toolbox root directory

Source code in toolboxv2/utils/tbx/setup.py
26
27
28
29
30
31
32
def get_tb_root(self) -> Path:
    """Get toolbox root directory"""
    try:
        from toolboxv2 import tb_root_dir
        return Path(tb_root_dir)
    except ImportError:
        return Path(__file__).parent.parent
setup_all()

Run complete setup

Source code in toolboxv2/utils/tbx/setup.py
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def setup_all(self):
    """Run complete setup"""
    print("╔════════════════════════════════════════════════════════════════╗")
    print("║         TB Language - System Integration Setup                 ║")
    print("╚════════════════════════════════════════════════════════════════╝")
    print()

    # Check prerequisites
    if not self.executable.exists():
        print("❌ TB executable not found!")
        print(f"   Expected at: {self.executable}")
        print("   Run 'tb x build' first!")
        return False

    print(f"✓ TB executable found: {self.executable}")
    print()

    # Setup icon
    if not self.setup_icon():
        print("⚠️  Icon setup failed (continuing anyway)")

    # Setup file association
    if self.system == "Windows":
        success = self.setup_windows()
    elif self.system == "Linux":
        success = self.setup_linux()
    elif self.system == "Darwin":
        success = self.setup_macos()
    else:
        print(f"❌ Unsupported system: {self.system}")
        return False

    if success:
        print()
        print("╔════════════════════════════════════════════════════════════════╗")
        print("║                    ✓ Setup Complete!                           ║")
        print("╠════════════════════════════════════════════════════════════════╣")
        print("║  .tbx files are now associated with TB Language                ║")
        print("║  Double-click any .tbx file to run it!                         ║")
        print("╚════════════════════════════════════════════════════════════════╝")

    return success
setup_icon()

Setup icon file

Source code in toolboxv2/utils/tbx/setup.py
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
    def setup_icon(self) -> bool:
        """Setup icon file"""
        print("📦 Setting up icon...")

        icon_dir = self.tb_root / "resources"
        icon_dir.mkdir(exist_ok=True)

        # Check if icon exists
        if self.icon_path.exists():
            print(f"   ✓ Icon already exists: {self.icon_path}")
            return True

        # Create placeholder icon info
        print(f"   ⚠️  Icon not found at: {self.icon_path}")
        print(f"   📝 Creating placeholder...")

        # Try to create a simple icon reference
        # User needs to provide actual tb_icon.ico file
        placeholder = icon_dir / "README_ICON.txt"
        placeholder.write_text("""
TB Language Icon
================

Place your icon files here:
- tb_icon.ico   (Windows)
- tb_icon.png   (Linux)
- tb_icon.icns  (macOS)

Recommended size: 256x256 px

You can use the ToolBox V2 logo/icon.
        """)

        print(f"   ℹ️  Place icon file at: {self.icon_path}")
        return False
setup_linux()

Setup file association on Linux

Source code in toolboxv2/utils/tbx/setup.py
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
    def setup_linux(self) -> bool:
        """Setup file association on Linux"""
        print("🐧 Setting up Linux file association...")

        try:
            # Create .desktop file
            desktop_dir = Path.home() / ".local" / "share" / "applications"
            desktop_dir.mkdir(parents=True, exist_ok=True)

            desktop_file = desktop_dir / "tb-language.desktop"

            icon_path = self.icon_path.with_suffix('.png')
            if not icon_path.exists():
                icon_path = "text-x-script"  # Fallback icon

            desktop_content = f"""[Desktop Entry]
Version=1.0
Type=Application
Name=TB Language
Comment=Execute TB Language programs
Exec={self.executable} run %f
Icon={icon_path}
Terminal=false
MimeType=text/x-tb;application/x-tb;
Categories=Development;
"""

            desktop_file.write_text(desktop_content)
            desktop_file.chmod(0o755)
            print(f"   ✓ Created desktop entry: {desktop_file}")

            # Create MIME type
            mime_dir = Path.home() / ".local" / "share" / "mime" / "packages"
            mime_dir.mkdir(parents=True, exist_ok=True)

            mime_file = mime_dir / "tb-language.xml"
            mime_content = """<?xml version="1.0" encoding="UTF-8"?>
<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
    <mime-type type="text/x-tb">
        <comment>TB Language Program</comment>
        <glob pattern="*.tbx"/>
        <sub-class-of type="text/plain"/>
    </mime-type>
</mime-info>
"""

            mime_file.write_text(mime_content)
            print(f"   ✓ Created MIME type: {mime_file}")

            # Update MIME database
            print("   Updating MIME database...")
            try:
                subprocess.run(["update-mime-database",
                                str(Path.home() / ".local" / "share" / "mime")],
                               check=True, capture_output=True)
                print("   ✓ MIME database updated")
            except:
                print("   ⚠️  Could not update MIME database automatically")
                print("   Run: update-mime-database ~/.local/share/mime")

            # Update desktop database
            print("   Updating desktop database...")
            try:
                subprocess.run(["update-desktop-database", str(desktop_dir)],
                               check=True, capture_output=True)
                print("   ✓ Desktop database updated")
            except:
                print("   ⚠️  Could not update desktop database automatically")

            # Set default application
            try:
                subprocess.run([
                    "xdg-mime", "default", "tb-language.desktop", "text/x-tb"
                ], check=True, capture_output=True)
                print("   ✓ Set as default application for .tbx files")
            except:
                print("   ⚠️  Could not set as default application")

            print("   ✓ Linux setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False
setup_macos()

Setup file association on macOS

Source code in toolboxv2/utils/tbx/setup.py
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
    def setup_macos(self) -> bool:
        """Setup file association on macOS"""
        print("🍎 Setting up macOS file association...")

        try:
            # Create Info.plist for file association
            app_dir = self.tb_root / "TB Language.app"
            contents_dir = app_dir / "Contents"
            macos_dir = contents_dir / "MacOS"
            resources_dir = contents_dir / "Resources"

            # Create directories
            macos_dir.mkdir(parents=True, exist_ok=True)
            resources_dir.mkdir(parents=True, exist_ok=True)

            # Copy executable
            app_executable = macos_dir / "tb"
            if not app_executable.exists():
                shutil.copy(self.executable, app_executable)
                app_executable.chmod(0o755)

            # Create launcher script
            launcher = macos_dir / "TB Language"
            launcher.write_text(f"""#!/bin/bash
if [ "$#" -gt 0 ]; then
    "{app_executable}" run "$@"
else
    "{app_executable}" repl
fi
""")
            launcher.chmod(0o755)

            # Create Info.plist
            plist_file = contents_dir / "Info.plist"
            plist_content = f"""<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
    <key>CFBundleExecutable</key>
    <string>TB Language</string>
    <key>CFBundleIconFile</key>
    <string>tb_icon</string>
    <key>CFBundleIdentifier</key>
    <string>dev.tblang.tb</string>
    <key>CFBundleName</key>
    <string>TB Language</string>
    <key>CFBundlePackageType</key>
    <string>APPL</string>
    <key>CFBundleShortVersionString</key>
    <string>1.0.0</string>
    <key>CFBundleVersion</key>
    <string>1.0.0</string>
    <key>CFBundleDocumentTypes</key>
    <array>
        <dict>
            <key>CFBundleTypeExtensions</key>
            <array>
                <string>tbx</string>
            </array>
            <key>CFBundleTypeIconFile</key>
            <string>tb_icon</string>
            <key>CFBundleTypeName</key>
            <string>TB Language Program</string>
            <key>CFBundleTypeRole</key>
            <string>Editor</string>
            <key>LSHandlerRank</key>
            <string>Owner</string>
        </dict>
    </array>
</dict>
</plist>
"""
            plist_file.write_text(plist_content)
            print(f"   ✓ Created app bundle: {app_dir}")

            # Copy icon if exists
            icon_src = self.icon_path.with_suffix('.icns')
            if icon_src.exists():
                shutil.copy(icon_src, resources_dir / "tb_icon.icns")
                print(f"   ✓ Copied icon")

            # Register with Launch Services
            print("   Registering with Launch Services...")
            try:
                subprocess.run([
                    "/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister",
                    "-f", str(app_dir)
                ], check=True, capture_output=True)
                print("   ✓ Registered with Launch Services")
            except:
                print("   ⚠️  Could not register automatically")
                print(f"   Run: open '{app_dir}'")

            print("   ✓ macOS setup complete!")
            return True

        except Exception as e:
            print(f"   ❌ Error: {e}")
            return False
setup_windows()

Setup file association on Windows

Source code in toolboxv2/utils/tbx/setup.py
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def setup_windows(self) -> bool:
    """Setup file association on Windows"""
    print("🪟 Setting up Windows file association...")

    try:
        import winreg

        # Create .tbx extension key
        print("   Creating registry entries...")

        # HKEY_CURRENT_USER\Software\Classes\.tbx
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TBLanguageFile")
            print("   ✓ Registered .tbx extension")

        # HKEY_CURRENT_USER\Software\Classes\TBLanguageFile
        with winreg.CreateKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile") as key:
            winreg.SetValue(key, "", winreg.REG_SZ, "TB Language Program")

            # Set icon
            if self.icon_path.exists():
                icon_key = winreg.CreateKey(key, "DefaultIcon")
                winreg.SetValue(icon_key, "", winreg.REG_SZ, str(self.icon_path))
                print(f"   ✓ Set icon: {self.icon_path}")

            # Set open command
            command_key = winreg.CreateKey(key, r"shell\open\command")
            cmd = f'"{self.executable}" run "%1"'
            winreg.SetValue(command_key, "", winreg.REG_SZ, cmd)
            print(f"   ✓ Set open command: {cmd}")

            # Add "Run in Terminal" context menu
            terminal_key = winreg.CreateKey(key, r"shell\run_terminal\command")
            terminal_cmd = f'cmd /k "{self.executable}" run "%1" && pause'
            winreg.SetValue(terminal_key, "", winreg.REG_SZ, terminal_cmd)
            winreg.SetValue(winreg.CreateKey(key, r"shell\run_terminal"), "", winreg.REG_SZ, "Run in Terminal")
            print(f"   ✓ Added 'Run in Terminal' context menu")

            # Add "Edit" context menu
            edit_key = winreg.CreateKey(key, r"shell\edit\command")
            winreg.SetValue(edit_key, "", winreg.REG_SZ, 'notepad "%1"')
            winreg.SetValue(winreg.CreateKey(key, r"shell\edit"), "", winreg.REG_SZ, "Edit")
            print(f"   ✓ Added 'Edit' context menu")

        # Refresh shell
        print("   Refreshing Explorer...")
        try:
            import ctypes
            ctypes.windll.shell32.SHChangeNotify(0x08000000, 0x0000, None, None)
        except:
            print("   ⚠️  Could not refresh Explorer (restart may be needed)")

        print("   ✓ Windows setup complete!")
        return True

    except ImportError:
        print("   ❌ winreg module not available")
        return False
    except Exception as e:
        print(f"   ❌ Error: {e}")
        return False
uninstall()

Remove file associations

Source code in toolboxv2/utils/tbx/setup.py
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
def uninstall(self):
    """Remove file associations"""
    print("🗑️  Uninstalling file associations...")

    if self.system == "Windows":
        try:
            import winreg
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\.tbx")
            winreg.DeleteKey(winreg.HKEY_CURRENT_USER, r"Software\Classes\TBLanguageFile")
            print("   ✓ Windows registry cleaned")
        except:
            print("   ⚠️  Could not clean registry")

    elif self.system == "Linux":
        desktop_file = Path.home() / ".local" / "share" / "applications" / "tb-language.desktop"
        mime_file = Path.home() / ".local" / "share" / "mime" / "packages" / "tb-language.xml"

        if desktop_file.exists():
            desktop_file.unlink()
            print("   ✓ Removed desktop entry")

        if mime_file.exists():
            mime_file.unlink()
            print("   ✓ Removed MIME type")

    elif self.system == "Darwin":
        app_dir = self.tb_root / "TB Language.app"
        if app_dir.exists():
            shutil.rmtree(app_dir)
            print("   ✓ Removed app bundle")

    print("   ✓ Uninstall complete!")
main()

Main entry point

Source code in toolboxv2/utils/tbx/setup.py
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
def main():
    """Main entry point"""
    import argparse

    parser = argparse.ArgumentParser(
        description="TB Language System Integration Setup"
    )
    parser.add_argument('action', choices=['install', 'uninstall'],
                        help='Action to perform')

    args = parser.parse_args()

    setup = TBxSetup()

    if args.action == 'install':
        success = setup.setup_all()
        sys.exit(0 if success else 1)
    elif args.action == 'uninstall':
        setup.uninstall()
        sys.exit(0)

test_tb_lang

TB Language Comprehensive Test Suite Tests all features of the TB language implementation.

Usage

python test_tb_lang.py python test_tb_lang.py --verbose python test_tb_lang.py --filter "test_arithmetic"

assert_contains(code, substring, mode='jit')

Assert that output contains substring.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
352
353
354
355
356
357
358
359
360
361
362
def assert_contains(code: str, substring: str, mode: str = "jit"):
    """Assert that output contains substring."""
    success, stdout, stderr = run_tb(code, mode)

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    if substring not in stdout:
        raise AssertionError(
            f"Output does not contain '{substring}':\n{stdout}"
        )
assert_output(code, expected, mode='jit')

Assert that TB code produces expected output.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
def assert_output(code: str, expected: str, mode: str = "jit"):
    """Assert that TB code produces expected output."""

    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
        print(f"stdout: {stdout}")
    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")

    actual = stdout.strip()
    expected = expected.strip()

    if actual != expected:
        raise AssertionError(
            f"Output mismatch:\n"
            f"Expected: {repr(expected)}\n"
            f"Got:      {repr(actual)}"
        )
assert_success(code, mode='jit')

Assert that TB code runs without error.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
338
339
340
341
342
343
344
345
346
347
348
349
def assert_success(code: str, mode: str = "jit"):
    """Assert that TB code runs without error."""
    success, stdout, stderr = run_tb(code, mode)
    if VERBOSE and not success:
        print(f"code: {code}")
        print()
    if VERBOSE:
        print(f"stdout: {stdout}")
        print(f"stderr: {stderr}")

    if not success:
        raise AssertionError(f"Execution failed:\n{stderr}")
find_tb_binary()

Find TB binary, trying multiple paths with system-specific extensions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def find_tb_binary() -> str:
    """Find TB binary, trying multiple paths with system-specific extensions."""
    from toolboxv2 import tb_root_dir
    # Path to TB binary (relative or absolute)
    TB_BINARY_PATH = str(tb_root_dir / "bin" / "tbx")

    # Alternative paths to try if main path doesn't exist
    ALTERNATIVE_PATHS = [
        str(tb_root_dir / "tb-exc" / "target" / "debug" / "tbx"),
        str(tb_root_dir / "tb-exc" / "target" / "release" / "tbx"),
        #"tbx",  # System PATH
    ]

    # Add system-specific extension
    def add_extension(path: str) -> str:
        if system() == "Windows" and not path.endswith(".exe"):
            return f"{path}.exe"
        return path

    # Prepare paths with proper extensions
    paths_to_try = [add_extension(TB_BINARY_PATH)]
    for alt_path in ALTERNATIVE_PATHS:
        paths_to_try.append(add_extension(alt_path))

    for path in paths_to_try:
        # Check if file exists directly
        if os.path.exists(path):
            return path

        # Use shutil.which for system PATH lookup (cross-platform)
        if shutil.which(path):
            return path

    print(f"{Colors.RED}✗ TB binary not found!{Colors.RESET}")
    print(f"{Colors.YELLOW}Tried paths:{Colors.RESET}")
    for path in paths_to_try:
        print(f"  • {path}")
    print(f"\n{Colors.CYAN}Build the binary with:{Colors.RESET}")
    print(f"  tb run build")
    return ""
run_tb(code, mode='jit', timeout=10)

Run TB code and return (success, stdout, stderr).

Parameters:

Name Type Description Default
code str

TB source code

required
mode str

Execution mode (jit, compiled, streaming)

'jit'
timeout int

Timeout in seconds

10

Returns:

Type Description
Tuple[bool, str, str]

(success, stdout, stderr)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def run_tb(code: str, mode: str = "jit", timeout: int = 10) -> Tuple[bool, str, str]:
    """
    Run TB code and return (success, stdout, stderr).

    Args:
        code: TB source code
        mode: Execution mode (jit, compiled, streaming)
        timeout: Timeout in seconds

    Returns:
        (success, stdout, stderr)
    """
    if mode == "compiled":
        with tempfile.NamedTemporaryFile(suffix='', delete=False) as f:
            output_path = f.name

        try:
            start = time.perf_counter()
            success, stderr = run_tb_compile(code, output_path)
            duration = time.perf_counter() - start
            print(f" -- Compile time ({duration:.3f}s)")
            if not success:
                raise AssertionError(f"Compilation failed:\n{stderr}")

            # Check binary exists
            if not os.path.exists(output_path):
                raise AssertionError("Compiled binary not found")

            # Check binary is executable
            if not os.access(output_path, os.X_OK):
                os.chmod(output_path, 0o755)

            start = time.perf_counter()
            # Run compiled binary
            result = subprocess.run([output_path], capture_output=True, text=True, timeout=timeout//2,
                                    encoding=sys.stdout.encoding or 'utf-8')
            duration = time.perf_counter() - start
            print(f" -- Exec time ({duration:.3f}s)")

            if result.returncode != 0:
                raise AssertionError(f"Compiled binary failed: {result.stderr}")

            return success, result.stdout, result.stderr
        finally:
            if os.path.exists(output_path):
                os.remove(output_path)

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        result = subprocess.run(
            [TB_BINARY, "run", temp_file, "--mode", mode],
            capture_output=True,
            text=True,
            timeout=timeout,
            encoding=sys.stdout.encoding or 'utf-8',
            errors='replace'
        )

        success = result.returncode == 0
        return success, result.stdout, result.stderr

    except subprocess.TimeoutExpired:
        return False, "", f"Timeout after {timeout}s"

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass
run_tb_compile(code, output_path, target=None)

Compile TB code to binary.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
def run_tb_compile(code: str, output_path: str, target: str = None) -> Tuple[bool, str]:
    """Compile TB code to binary."""
    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, encoding=sys.stdout.encoding or 'utf-8') as f:
        f.write(code)
        temp_file = f.name

    try:
        cmd = [TB_BINARY, "compile", temp_file, output_path]
        if target:
            cmd.extend(["--target", target])

        result = subprocess.run(cmd, capture_output=not VERBOSE, text=True, timeout=60, encoding=sys.stdout.encoding or 'utf-8')

        success = result.returncode == 0
        return success, result.stderr

    finally:
        try:
            os.unlink(temp_file)
        except:
            pass
test(name, category='General')

Decorator for test functions.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
def test(name: str, category: str = "General"):
    """Decorator for test functions."""

    def decorator(func):
        def wrapper():
            # Check filter
            if FILTER and FILTER.lower() not in name.lower():
                return

            # Print test header
            if suite.current_category != category:
                print(f"\n{Colors.BOLD}{Colors.CYAN}[{category}]{Colors.RESET}")
                suite.current_category = category

            print(f"  {Colors.GRAY}Testing:{Colors.RESET} {name}", end=" ", flush=True)

            start = time.perf_counter()
            try:
                func()
                duration = time.perf_counter() - start

                print(f" -- {Colors.GREEN}{Colors.RESET} ({duration:.3f}s)")

                suite.add_result(TestResult(
                    name=name,
                    passed=True,
                    duration_ms=duration
                ))

            except AssertionError as e:
                duration = time.perf_counter() - start

                print(f"{Colors.RED}{Colors.RESET} ({duration:.0f}s)")
                if VERBOSE:
                    print(f"    {Colors.RED}Error: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=str(e)
                ))

            except Exception as e:
                import traceback
                print(traceback.format_exc())
                duration = (time.perf_counter() - start) * 1000

                print(f"{Colors.RED}✗ (Exception){Colors.RESET}")
                if VERBOSE:
                    print(f"    {Colors.RED}{type(e).__name__}: {str(e)}{Colors.RESET}")

                suite.add_result(TestResult(
                    name=name,
                    passed=False,
                    duration_ms=duration,
                    error_message=f"{type(e).__name__}: {str(e)}"
                ))

        return wrapper

    return decorator
test_compiled_arithmetic()

Test basic arithmetic in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
@test("Compiled - Basic Arithmetic", "Compiled Modes")
def test_compiled_arithmetic():
    """Test basic arithmetic in compiled mode"""
    code = '''
let x = 10
let y = 20
let sum = x + y
let product = x * y
echo sum
echo product
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    assert "30" in stdout
    assert "200" in stdout
test_compiled_functions()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
@test("Compiled - Functions", "Compiled Modes")
def test_compiled_functions():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="compiled")
test_compiled_functions_no_compiled_()

Test function definitions in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
@test("Compiled - Functions", "Compiled Modes NO Compiled")
def test_compiled_functions_no_compiled_():
    """Test function definitions in compiled mode"""
    code = '''
fn fibonacci(n: int) -> int {
    if n <= 1 {
        n
    } else {
        fibonacci(n - 1) + fibonacci(n - 2)
    }
}

let result = fibonacci(10)
echo result
'''
    assert_output(code, "55", mode="jit")
test_compiled_loops()

Test loops in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
@test("Compiled - Loops", "Compiled Modes")
def test_compiled_loops():
    """Test loops in compiled mode"""
    code = '''
let sum = 0
for i in [1, 2, 3, 4, 5] {
    sum = sum + i
}
echo sum
'''
    assert_output(code, "15", mode="compiled")
test_compiled_parallel()

Test parallel execution in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
@test("Compiled - Parallel Execution", "Compiled Modes")
def test_compiled_parallel():
    """Test parallel execution in compiled mode"""
    code = '''
@config {
    runtime_mode: "parallel"
}

let results = parallel {
    10 + 5,
    20 * 2,
    30 - 10
}
for result in results {
    echo result
}
'''
    success, stdout, stderr = run_tb(code, mode="compiled")
    assert success, f"Compilation failed: {stderr}"
    # Results may be in any order due to parallelism
    assert "15" in stdout, f"result {stdout}"
    assert "40" in stdout, f"result {stdout}"
    assert "20" in stdout, f"result {stdout}"
test_compiled_parallel_with_imports()

Test parallel execution with imported functions

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
@test("Compiled - Parallel with Imports", "Compiled Modes")
def test_compiled_parallel_with_imports():
    """Test parallel execution with imported functions"""
    lib_code = '''
fn compute_square(x: int) {
    x * x
}

fn compute_cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{lib_path}"
}}

let results = parallel {{
    compute_square(5),
    compute_cube(3),
    compute_square(10)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "25" in stdout
        assert "27" in stdout
        assert "100" in stdout
    finally:
        os.unlink(lib_path)
test_go_basic_execution()

Test that Go code can be executed.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
@test("Go Language Bridge - Basic Execution", "Dependencies - Go Bridge")
def test_go_basic_execution():
    """Test that Go code can be executed."""
    code = '''
let result = go("""
import "fmt"
fmt.Println("Hello from Go!")
""")
'''
    assert_success(code)
test_go_bridge_compiled_mode()

Test that the Go language bridge works in compiled mode.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
@test("Go Bridge - Compiled Mode", "Dependencies - Go Bridge")
def test_go_bridge_compiled_mode():
    """Test that the Go language bridge works in compiled mode."""
    code = '''
@config {
    mode: "compiled"
}

let result = go("""
import "fmt"
fmt.Println("Hello from Go in compiled mode!")
""")

echo "Go execution finished."
'''
    assert_contains(code, "Go execution finished.", mode="compiled")
test_go_import_json()

Test marshalling data to JSON using Go's 'encoding/json' package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
@test("Go Import - JSON Marshalling", "Dependencies - Go Bridge")
def test_go_import_json():
    """Test marshalling data to JSON using Go's 'encoding/json' package."""
    code = '''
# This test assumes TB lists/dicts are converted to Go slices/maps
let user_data = [
    { "name": "Alice", "age": 30 },
    { "name": "Bob", "age": 25 }
]
let result = go("""
import (
    "fmt"
    "encoding/json"
)
// user_data is available from TB, assume it's a []map[string]interface{}
// Marshal returns the JSON encoding of user_data.
jsonData, err := json.Marshal(user_data)
if err != nil {
    fmt.Println("Error:", err)
} else {
    fmt.Println(string(jsonData))
}
""")
'''
    assert_success(code)
test_go_import_math()

Test using the Go 'math' standard library package.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
@test("Go Import - Math", "Dependencies - Go Bridge")
def test_go_import_math():
    """Test using the Go 'math' standard library package."""
    code = '''
let result = go("""
import (
    "fmt"
    "math"
)
// Sqrt returns the square root of 64
fmt.Println(math.Sqrt(64))
""")
'''
    assert_success(code)
test_go_import_strings()

Test using the Go 'strings' package with a TB variable.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
@test("Go Import - Strings", "Dependencies - Go Bridge")
def test_go_import_strings():
    """Test using the Go 'strings' package with a TB variable."""
    code = '''
let my_string = "Hello, Go!"
let result = go("""
import (
    "fmt"
    "strings"
)
// my_string is available from TB context
// ToUpper returns a copy of the string with all letters mapped to their upper case.
fmt.Println(strings.ToUpper(my_string))
""")
'''
    assert_success(code)
test_go_multilang_python_to_go()

Test data flow from a Python block to a Go block.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
@test("Go Multi-Language - Python to Go", "Dependencies - Go Bridge")
def test_go_multilang_python_to_go():
    """Test data flow from a Python block to a Go block."""
    code = '''
# Python creates a string
let data_from_py = python("""
print("Data generated by Python")
""")

# Go processes the string from the TB variable
let go_result = go("""
import "fmt"
// data_from_py is available from TB context
// Printf formats according to a format specifier and writes to standard output.
fmt.Printf("Go received: '%s'\\n", data_from_py)
""")
'''
    assert_success(code)
test_import_cache_invalidation()

Test that cache is invalidated when source changes

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
@test("Compiled - Cache Invalidation", "Import Compiled Modes")
def test_import_cache_invalidation():
    """Test that cache is invalidated when source changes"""
    print("→ Test: Cache Invalidation")

    lib_path = 'test_cache_lib.tbx'

    # Write initial version
    with open(lib_path, 'w') as f:
        f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    42
}
''')

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo get_value()
'''

        # First run
        success1, stdout1, _ = run_tb(main_code)
        assert success1
        assert "42" in stdout1

        # Modify library
        time.sleep(0.1)  # Ensure timestamp difference
        with open(lib_path, 'w') as f:
            f.write('''
@config {
    mode: "compiled"
}

fn get_value() {
    100
}
''')

        # Second run should use updated version
        success2, stdout2, _ = run_tb(main_code)
        assert success2
        assert "100" in stdout2, f"Cache not invalidated! Got: {stdout2}"

        print("  ✓ Cache invalidation works")

    finally:
        os.unlink(lib_path)
test_import_compiled_caching()

Test that compiled imports are cached

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
@test("Compiled - Import Caching", "Import Compiled Modes")
def test_import_compiled_caching():
    """Test that compiled imports are cached"""
    print("→ Test: Compiled Import Caching")

    # Create library with compiled mode
    lib_code = '''
@config {
    mode: "compiled"
}

fn factorial(n: int) {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = factorial(5)
echo $result
'''

        # First run - should compile
        start_time = time.perf_counter()
        success1, stdout1, stderr1 = run_tb(main_code)
        first_run_time = time.perf_counter() - start_time

        assert success1, f"First execution failed: {stderr1}"
        assert "120" in stdout1, f"Expected 120, got: {stdout1}"

        # Second run - should use cache
        start_time = time.perf_counter()
        success2, stdout2, stderr2 = run_tb(main_code)
        second_run_time = time.perf_counter() - start_time

        assert success2, f"Second execution failed: {stderr2}"
        assert "120" in stdout2, f"Expected 120, got: {stdout2}"

        # Second run should be significantly faster (using cache)
        assert second_run_time <= first_run_time, \
            f"Cache not used? First: {first_run_time:.2f}s, Second: {second_run_time:.2f}s"

        print(f"  ✓ Caching works (first: {first_run_time:.2f}s, second: {second_run_time:.2f}s)")

    finally:
        os.unlink(lib_path)
test_import_compiled_mode()

Test that imports work in compiled mode

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
@test("Import - Compiled Mode", "Import System")
def test_import_compiled_mode():
    """Test that imports work in compiled mode"""
    lib_code = '''
@config {
    mode: "compiled"
    target: "library"
}

fn double(x: int) {
    x * 2
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "compiled"
}}

@imports {{
    "{lib_path}"
}}

let result = double(21)
echo result
'''
        assert_output(main_code, "42", mode="compiled")
    finally:
        os.unlink(lib_path)
test_import_dependency_chain()

Test imports that depend on other imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
@test("Compiled - Import Dependency Chain", "Import Compiled Modes")
def test_import_dependency_chain():
    """Test imports that depend on other imports"""
    print("→ Test: Import Dependency Chain")

    # Base library
    base_lib = '''
fn double(x: int) {
    x * 2
}
'''

    # Mid library (uses base)
    mid_lib = '''
@imports {
    "base.tbx"
}

fn quadruple(x: int) {
    double(double(x))
}
'''

    # Create base.tbx
    with open('base.tbx', 'w') as f:
        f.write(base_lib)

    # Create mid.tbx
    with open('mid.tbx', 'w') as f:
        f.write(mid_lib)

    try:
        main_code = '''
@imports {
    "mid.tbx"
}

echo quadruple(5)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "20" in stdout, f"Expected 20, got: {stdout}"

        print("  ✓ Transitive imports work")

    finally:
        os.unlink('base.tbx')
        os.unlink('mid.tbx')
test_import_jit_vs_compiled()

Test that JIT imports don't get compiled

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
@test("Compiled - JIT vs Compiled Imports", "Import Compiled Modes")
def test_import_jit_vs_compiled():
    """Test that JIT imports don't get compiled"""
    print("→ Test: JIT vs Compiled Imports")

    # JIT library
    jit_lib = '''
@config {
    mode: "jit"
}

fn square(x: int) {
    x * x
}
'''

    # Compiled library
    compiled_lib = '''
@config {
    mode: "compiled"
}

fn cube(x: int) {
    x * x * x
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as jit_file, \
         tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as compiled_file:

        jit_file.write(jit_lib)
        compiled_file.write(compiled_lib)
        jit_path = os.path.basename(jit_file.name)
        compiled_path = os.path.basename(compiled_file.name)

    try:
        main_code = f'''
@imports {{
    "{jit_path}"
    "{compiled_path}"
}}

echo square(4)
echo cube(3)
'''

        success, stdout, stderr = run_tb(main_code)

        assert success, f"Execution failed: {stderr}"
        assert "16" in stdout, f"Expected 16 from square(4), got: {stdout}"
        assert "27" in stdout, f"Expected 27 from cube(3), got: {stdout}"

        print("  ✓ Mixed JIT/Compiled imports work")

    finally:
        os.unlink(jit_path)
        os.unlink(compiled_path)
test_import_missing_file()

Test error handling for missing import files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
@test("Import - Missing File Error", "Import Errors")
def test_import_missing_file():
    """Test error handling for missing import files"""
    code = '''
@imports {
    "nonexistent_file.tbx"
}

echo "This should not run"
'''
    success, stdout, stderr = run_tb(code)
    assert not success, "Should fail for missing import file"
    assert "not found" in stderr.lower() or "import" in stderr.lower()
test_import_multiple_files()

Test importing multiple .tbx files

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
@test("Import - Multiple Files", "Import System")
def test_import_multiple_files():
    """Test importing multiple .tbx files"""
    # Create math library
    math_lib = '''
fn add(x, y) {
    x + y
}
'''

    # Create string library
    string_lib = '''
fn greet(name: string) {
    echo "Hello,"name "!"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as string_file:

        math_file.write(math_lib)
        string_file.write(string_lib)
        math_path = os.path.basename(math_file.name)
        string_path = os.path.basename(string_file.name)

    try:
        main_code = f'''
@imports {{
    "{math_path}"
    "{string_path}"
}}

let sum = add(3, 7)
echo sum
greet("World")
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "10" in stdout, f"Expected '10' in output, got: {stdout}"
        assert "Hello, World !" in stdout, f"Expected Hello, World! in output, got: {stdout}"
    finally:
        os.unlink(math_path)
        os.unlink(string_path)
test_import_no_circular()

Test that circular imports are handled (imports are not recursive)

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
@test("Import - Circular Import Protection", "Import Errors")
def test_import_no_circular():
    """Test that circular imports are handled (imports are not recursive)"""
    # Create file that tries to import itself
    lib_code = '''
fn test_func() {
    echo "Hello"
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # This should work because imports are not recursive
        main_code = f'''
@imports {{
    "{lib_path}"
}}

test_func()
'''
        assert_success(main_code)
    finally:
        os.unlink(lib_path)
test_import_relative_paths()

Test importing with relative directory paths

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
@test("Import - Relative Paths", "Import System")
def test_import_relative_paths():
    """Test importing with relative directory paths"""
    # Create subdirectory
    lib_dir = tempfile.mkdtemp(dir='..')
    lib_dir_name = os.path.basename(lib_dir)

    try:
        # Create library in subdirectory
        lib_code = '''
fn multiply(x: int, y: int) {
    x * y
}
'''
        lib_path = os.path.join(lib_dir, 'math.tbx')
        with open(lib_path, 'w') as f:
            f.write(lib_code)

        main_code = f'''
@imports {{
    "{lib_dir_name}/math.tbx"
}}

let result = multiply(6, 7)
echo result
'''
        assert_output(main_code, "42")
    finally:
        shutil.rmtree(lib_dir, ignore_errors=True)
test_import_single_file()

Test importing a single .tbx file

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
@test("Import - Single File", "Import System")
def test_import_single_file():
    """Test importing a single .tbx file"""
    # Create library file
    lib_code = '''
fn double(x: int) {
    x * 2
}

fn triple(x: int) {
    x * 3
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        # Create main file that imports library
        main_code = f'''
@imports {{
    "{lib_path}"
}}

let result = double(5)
echo result
'''
        assert_output(main_code, "10")
    finally:
        os.unlink(lib_path)
test_import_with_variables()

Test importing file with global variables

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
@test("Import - With Variables", "Import System")
def test_import_with_variables():
    """Test importing file with global variables"""
    lib_code = '''
let PI = 3.14159
let E = 2.71828

fn circle_area(radius: float) {
    PI * radius * radius
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@imports {{
    "{lib_path}"
}}

echo PI
let area = circle_area(2.0)
echo area
'''
        success, stdout, stderr = run_tb(main_code)
        assert success, f"Execution failed: {stderr}"
        assert "3.14159" in stdout or "3.14" in stdout
    finally:
        os.unlink(lib_path)
test_mixed_full_stack()

Comprehensive test with all features combined

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
@test("Mixed - Full Stack Test", "Mixed Features")
def test_mixed_full_stack():
    """Comprehensive test with all features combined"""
    helpers_lib = '''
fn compute(x: int) {
    x * 2
}

fn parallel_sum(numbers: list) {
    let sum = 0
    for n in numbers {
        sum = sum + n
    }
    sum
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(helpers_lib)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@config {{
    mode: "jit"
}}

@shared {{
    total: 0
}}

@imports {{
    "{lib_path}"
}}

let value = compute(21)
total = parallel_sum([1, 2, 3, 4, 5])
echo value
echo total
'''
        success, stdout, stderr = run_tb(main_code, mode="jit")
        assert success, f"Execution failed: {stderr}"
        assert "42" in stdout
        assert "15" in stdout
    finally:
        os.unlink(lib_path)
test_mixed_import_async_language()

Test imports with async and language bridges

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
@test("Mixed - Import  + Language Bridge", "Mixed Features")
def test_mixed_import_async_language():
    """Test imports with async and language bridges"""
    lib_code = '''
fn get_data() {

    python("import sys; print(sys.version.split()[0])")

}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''

@imports {{
    "{lib_path}"
}}

let version = get_data()
echo "Python version detected"
'''
        assert_contains(main_code, "Python version detected", mode="jit")
    finally:
        os.unlink(lib_path)
test_mixed_multiple_imports_parallel()

Test multiple imports with parallel execution

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
@test("Mixed - Multiple Imports + Parallel", "Mixed Features")
def test_mixed_multiple_imports_parallel():
    """Test multiple imports with parallel execution"""
    math_lib = '''
fn factorial(n: int) -> int {
    if n <= 1 {
        1
    } else {
        n * factorial(n - 1)
    }
}
'''

    utils_lib = '''
fn power(base: int, exp: int) -> int {
    if exp == 0 {
        1
    } else {
        base * power(base, exp - 1)
    }
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as math_file, \
        tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as utils_file:

        math_file.write(math_lib)
        utils_file.write(utils_lib)
        math_path = os.path.basename(math_file.name)
        utils_path = os.path.basename(utils_file.name)

    try:
        main_code = f'''
@config {{
    runtime_mode: "parallel"
}}

@imports {{
    "{math_path}"
    "{utils_path}"
}}

let results = parallel {{
    factorial(5),
    power(2, 10),
    factorial(6)
}}

for result in results {{
    echo result
}}
'''
        success, stdout, stderr = run_tb(main_code, mode="compiled")
        assert success, f"Compilation failed: {stderr}"
        assert "120" in stdout  # 5!
        assert "1024" in stdout  # 2^10
        assert "720" in stdout  # 6!
    finally:
        os.unlink(math_path)
        os.unlink(utils_path)
test_mixed_shared_imports()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
@test("Mixed - Shared Variables + Imports", "Mixed Features")
def test_mixed_shared_imports():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {
    counter + 1
}
'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)
test_mixed_shared_imports_mixed_lang()

Test shared variables with imports

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
@test("Mixed - Shared Variables + Imports + Python Bridge", "Mixed Features")
def test_mixed_shared_imports_mixed_lang():
    """Test shared variables with imports"""
    lib_code = '''
fn increment_counter() {python("print(counter + 1, end='')")}

'''

    with tempfile.NamedTemporaryFile(mode='w', suffix='.tbx', delete=False, dir='..') as lib_file:
        lib_file.write(lib_code)
        lib_path = os.path.basename(lib_file.name)

    try:
        main_code = f'''
@shared {{
    counter: 0
}}

@imports {{
    "{lib_path}"
}}

counter = increment_counter()
counter = increment_counter()
echo counter
'''
        assert_output(main_code, "2")
    finally:
        os.unlink(lib_path)
test_type_annotations()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("""x = "Alice"
x""")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")
test_type_annotations_()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
@test("Type Annotations - Basic Types", "Type Annotations")
def test_type_annotations_():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("19.99")
let name: string = python("Alice")
let active: bool = python("True")
let scores: list<int> = python("[85, 92, 78]")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true")
test_type_annotations_auto()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
@test("Type Annotations - Auto Type Inference", "Type Annotations")
def test_type_annotations_auto():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "jit", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo type_of(age)           // Age: 42
echo type_of(price)       // Price: 19.99
echo type_of(name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist")
test_type_annotations_auto_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
@test("Type Annotations - Auto Type Inference  - Compiled", "Type Annotations - Compiled")
def test_type_annotations_auto_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list<int> = python("print([85, 92, 78])")

echo type_of($age)           // Age: 42
echo type_of($price)       // Price: 19.99
echo type_of($name)         // Name: Alice
echo type_of(active)         // Active: true
echo type_of(scores)     // Scores: [85, 92, 78]'''
    assert_output(code, "int\nfloat\nstring\nbool\nlist<int>", mode="compiled")
test_type_annotations_compiled()

Test that type annotations are correctly handled.

Source code in toolboxv2/utils/tbx/test/test_tb_lang.py
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
@test("Type Annotations - Basic Type - Compiled", "Type Annotations - Compiled")
def test_type_annotations_compiled():
    """Test that type annotations are correctly handled."""
    code = '''@config { mode: "compiled", type_system: "static" }

let age: int = python("print(42)")
let price: float = python("print(19.99)")
let name: string = python("print('Alice')")
let active: bool = python("print(True)")
let scores: list = python("print([85, 92, 78])")

echo "Age: $age"           // Age: 42
echo "Price: $price"       // Price: 19.99
echo "Name: $name"         // Name: Alice
echo "Active: $active"     // Active: true'''
    assert_output(code, "Age: 42\nPrice: 19.99\nName: Alice\nActive: true", mode="compiled")

toolbox

Main module.

App
Source code in toolboxv2/utils/toolbox.py
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
class App(AppType, metaclass=Singleton):

    def __init__(self, prefix: str = "", args=AppArgs().default()):
        if "test" not in prefix:
            prefix = "main"
        super().__init__(prefix, args)
        self._web_context = None
        t0 = time.perf_counter()
        abspath = os.path.abspath(__file__)
        self.system_flag = system()  # Linux: Linux Mac: Darwin Windows: Windows

        self.appdata = os.getenv('APPDATA') if os.name == 'nt' else os.getenv('XDG_CONFIG_HOME') or os.path.expanduser(
                '~/.config') if os.name == 'posix' else None

        if self.system_flag == "Darwin" or self.system_flag == "Linux":
            dir_name = os.path.dirname(abspath).replace("/utils", "")
        else:
            dir_name = os.path.dirname(abspath).replace("\\utils", "")

        self.start_dir = str(dir_name)

        self.bg_tasks = []

        lapp = dir_name + '\\.data\\'

        if not prefix:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt") as prefix_file:
                cont = prefix_file.read()
                if cont:
                    prefix = cont.rstrip()
        else:
            if not os.path.exists(f"{lapp}last-app-prefix.txt"):
                os.makedirs(lapp, exist_ok=True)
                open(f"{lapp}last-app-prefix.txt", "a").close()
            with open(f"{lapp}last-app-prefix.txt", "w") as prefix_file:
                prefix_file.write(prefix)

        self.prefix = prefix

        node_ = node()

        if 'localhost' in node_ and (host := os.getenv('HOSTNAME', 'localhost')) != 'localhost':
            node_ = node_.replace('localhost', host)
        self.id = prefix + '-' + node_
        self.globals = {
            "root": {**globals()},
        }
        self.locals = {
            "user": {'app': self, **locals()},
        }

        identification = self.id
        collective_identification = self.id
        if "test" in prefix:
            if self.system_flag == "Darwin" or self.system_flag == "Linux":
                start_dir = self.start_dir.replace("ToolBoxV2/toolboxv2", "toolboxv2")
            else:
                start_dir = self.start_dir.replace("ToolBoxV2\\toolboxv2", "toolboxv2")
            self.data_dir = start_dir + '\\.data\\' + "test"
            self.config_dir = start_dir + '\\.config\\' + "test"
            self.info_dir = start_dir + '\\.info\\' + "test"
        elif identification.startswith('collective-'):
            collective_identification = identification.split('-')[1]
            self.data_dir = self.start_dir + '\\.data\\' + collective_identification
            self.config_dir = self.start_dir + '\\.config\\' + collective_identification
            self.info_dir = self.start_dir + '\\.info\\' + collective_identification
            self.id = collective_identification
        else:
            self.data_dir = self.start_dir + '\\.data\\' + identification
            self.config_dir = self.start_dir + '\\.config\\' + identification
            self.info_dir = self.start_dir + '\\.info\\' + identification

        if self.appdata is None:
            self.appdata = self.data_dir
        else:
            self.appdata += "/ToolBoxV2"

        if not os.path.exists(self.appdata):
            os.makedirs(self.appdata, exist_ok=True)
        if not os.path.exists(self.data_dir):
            os.makedirs(self.data_dir, exist_ok=True)
        if not os.path.exists(self.config_dir):
            os.makedirs(self.config_dir, exist_ok=True)
        if not os.path.exists(self.info_dir):
            os.makedirs(self.info_dir, exist_ok=True)

        self.print(f"Starting ToolBox as {prefix} from :", Style.Bold(Style.CYAN(f"{os.getcwd()}")))

        logger_info_str, self.logger, self.logging_filename = self.set_logger(args.debug)

        self.print("Logger " + logger_info_str)
        self.print("================================")
        self.logger.info("Logger initialized")
        get_logger().info(Style.GREEN("Starting Application instance"))
        if args.init and args.init is not None and self.start_dir not in sys.path:
            sys.path.append(self.start_dir)

        __version__ = get_version_from_pyproject()
        self.version = __version__

        self.keys = {
            "MACRO": "macro~~~~:",
            "MACRO_C": "m_color~~:",
            "HELPER": "helper~~~:",
            "debug": "debug~~~~:",
            "id": "name-spa~:",
            "st-load": "mute~load:",
            "comm-his": "comm-his~:",
            "develop-mode": "dev~mode~:",
            "provider::": "provider::",
        }

        defaults = {
            "MACRO": ['Exit'],
            "MACRO_C": {},
            "HELPER": {},
            "debug": args.debug,
            "id": self.id,
            "st-load": False,
            "comm-his": [[]],
            "develop-mode": False,
        }
        self.config_fh = FileHandler(collective_identification + ".config", keys=self.keys, defaults=defaults)
        self.config_fh.load_file_handler()
        self._debug = args.debug
        self.flows = {}
        self.dev_modi = self.config_fh.get_file_handler(self.keys["develop-mode"])
        if self.config_fh.get_file_handler("provider::") is None:
            self.config_fh.add_to_save_file_handler("provider::", "http://localhost:" + str(
                self.args_sto.port) if os.environ.get("HOSTNAME","localhost") == "localhost" else "https://simplecore.app")
        self.functions = {}
        self.modules = {}

        self.interface_type = ToolBoxInterfaces.native
        self.PREFIX = Style.CYAN(f"~{node()}@>")
        self.alive = True
        self.called_exit = False, time.time()

        self.print(f"Infos:\n  {'Name':<8} -> {node()}\n  {'ID':<8} -> {self.id}\n  {'Version':<8} -> {self.version}\n")

        self.logger.info(
            Style.GREEN(
                f"Finish init up in {time.perf_counter() - t0:.2f}s"
            )
        )

        self.args_sto = args
        self.loop = None

        from .system.session import Session
        self.session: Session = Session(self.get_username())
        if len(sys.argv) > 2 and sys.argv[1] == "db":
            return
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager, get_executable_path
        self.cluster_manager = ClusterManager()
        online_list, server_list = self.cluster_manager.status_all(silent=True)
        if not server_list:
            self.cluster_manager.start_all(get_executable_path(), self.version)
            _, server_list = self.cluster_manager.status_all()
        from .extras.blobs import BlobStorage
        self.root_blob_storage = BlobStorage(servers=server_list, storage_directory=self.data_dir+ '\\blob_cache\\')
        self.mkdocs = add_to_app(self)
        # self._start_event_loop()

    def _start_event_loop(self):
        """Starts the asyncio event loop in a separate thread."""
        if self.loop is None:
            self.loop = asyncio.new_event_loop()
            self.loop_thread = threading.Thread(target=self.loop.run_forever, daemon=True)
            self.loop_thread.start()

    def get_username(self, get_input=False, default="loot") -> str:
        user_name = self.config_fh.get_file_handler("ac_user:::")
        if get_input and user_name is None:
            user_name = input("Input your username: ")
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        if user_name is None:
            user_name = default
            self.config_fh.add_to_save_file_handler("ac_user:::", user_name)
        return user_name

    def set_username(self, username):
        return self.config_fh.add_to_save_file_handler("ac_user:::", username)

    @staticmethod
    def exit_main(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def hide_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def show_console(*args, **kwargs):
        """proxi attr"""

    @staticmethod
    def disconnect(*args, **kwargs):
        """proxi attr"""

    def set_logger(self, debug=False):
        if "test" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.NOTSET, name="toolbox-test", interminal=True,
                                                     file_level=logging.NOTSET, app_name=self.id)
            logger_info_str = "in Test Mode"
        elif "live" in self.prefix and not debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-live", interminal=False,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in Live Mode"
            # setup_logging(logging.WARNING, name="toolbox-live", is_online=True
            #              , online_level=logging.WARNING).info("Logger initialized")
        elif "debug" in self.prefix or self.prefix.endswith("D"):
            self.prefix = self.prefix.replace("-debug", '').replace("debug", '')
            logger, logging_filename = setup_logging(logging.DEBUG, name="toolbox-debug", interminal=True,
                                                     file_level=logging.WARNING, app_name=self.id)
            logger_info_str = "in debug Mode"
            self.debug = True
        elif debug:
            logger, logging_filename = setup_logging(logging.DEBUG, name=f"toolbox-{self.prefix}-debug",
                                                     interminal=True,
                                                     file_level=logging.DEBUG, app_name=self.id)
            logger_info_str = "in args debug Mode"
        else:
            logger, logging_filename = setup_logging(logging.ERROR, name=f"toolbox-{self.prefix}", app_name=self.id)
            logger_info_str = "in Default"

        return logger_info_str, logger, logging_filename

    @property
    def debug(self):
        return self._debug

    @debug.setter
    def debug(self, value):
        if not isinstance(value, bool):
            self.logger.debug(f"Value must be an boolean. is : {value} type of {type(value)}")
            raise ValueError("Value must be an boolean.")

        # self.logger.info(f"Setting debug {value}")
        self._debug = value

    def debug_rains(self, e):
        if self.debug:
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)
            raise e
        else:
            self.logger.error(f"Error: {e}")
            import traceback
            x = "="*5
            x += " DEBUG "
            x += "="*5
            self.print(x)
            self.print(traceback.format_exc())
            self.print(x)

    def set_flows(self, r):
        self.flows = r

    async def run_flows(self, name, **kwargs):
        from ..flows import flows_dict as flows_dict_func
        if name not in self.flows:
            self.flows = {**self.flows, **flows_dict_func(s=name, remote=True)}
        if name in self.flows:
            if asyncio.iscoroutinefunction(self.flows[name]):
                return await self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
            else:
                return self.flows[name](get_app(from_="runner"), self.args_sto, **kwargs)
        else:
            print("Flow not found, active flows:", len(self.flows.keys()))

    def _coppy_mod(self, content, new_mod_dir, mod_name, file_type='py'):

        mode = 'xb'
        self.logger.info(f" coppy mod {mod_name} to {new_mod_dir} size : {sys.getsizeof(content) / 8388608:.3f} mb")

        if not os.path.exists(new_mod_dir):
            os.makedirs(new_mod_dir)
            with open(f"{new_mod_dir}/__init__.py", "w") as nmd:
                nmd.write(f"__version__ = '{self.version}'")

        if os.path.exists(f"{new_mod_dir}/{mod_name}.{file_type}"):
            mode = False

            with open(f"{new_mod_dir}/{mod_name}.{file_type}", 'rb') as d:
                runtime_mod = d.read()  # Testing version but not efficient

            if len(content) != len(runtime_mod):
                mode = 'wb'

        if mode:
            with open(f"{new_mod_dir}/{mod_name}.{file_type}", mode) as f:
                f.write(content)

    def _pre_lib_mod(self, mod_name, path_to="./runtime", file_type='py'):
        working_dir = self.id.replace(".", "_")
        lib_mod_dir = f"toolboxv2.runtime.{working_dir}.mod_lib."

        self.logger.info(f"pre_lib_mod {mod_name} from {lib_mod_dir}")

        postfix = "_dev" if self.dev_modi else ""
        mod_file_dir = f"./mods{postfix}/{mod_name}.{file_type}"
        new_mod_dir = f"{path_to}/{working_dir}/mod_lib"
        with open(mod_file_dir, "rb") as c:
            content = c.read()
        self._coppy_mod(content, new_mod_dir, mod_name, file_type=file_type)
        return lib_mod_dir

    def _copy_load(self, mod_name, file_type='py', **kwargs):
        loc = self._pre_lib_mod(mod_name, file_type)
        return self.inplace_load_instance(mod_name, loc=loc, **kwargs)

    def helper_install_pip_module(self, module_name):
        if 'main' in self.id:
            return
        self.print(f"Installing {module_name} GREEDY")
        os.system(f"{sys.executable} -m pip install {module_name}")

    def python_module_import_classifier(self, mod_name, error_message):

        if error_message.startswith("No module named 'toolboxv2.utils"):
            return Result.default_internal_error(f"404 {error_message.split('utils')[1]} not found")
        if error_message.startswith("No module named 'toolboxv2.mods"):
            if mod_name.startswith('.'):
                return
            return self.run_a_from_sync(self.a_run_any, ("CloudM", "install"), module_name=mod_name)
        if error_message.startswith("No module named '"):
            pip_requ = error_message.split("'")[1].replace("'", "").strip()
            # if 'y' in input(f"\t\t\tAuto install {pip_requ} Y/n").lower:
            return self.helper_install_pip_module(pip_requ)
            # return Result.default_internal_error(f"404 {pip_requ} not found")

    def inplace_load_instance(self, mod_name, loc="toolboxv2.mods.", spec='app', save=True, mfo=None):
        if self.dev_modi and loc == "toolboxv2.mods.":
            loc = "toolboxv2.mods_dev."
        if spec=='app' and self.mod_online(mod_name):
            self.logger.info(f"Reloading mod from : {loc + mod_name}")
            self.remove_mod(mod_name, spec=spec, delete=False)

        if (os.path.exists(self.start_dir + '/mods/' + mod_name) or os.path.exists(
            self.start_dir + '/mods/' + mod_name + '.py')) and (
            os.path.isdir(self.start_dir + '/mods/' + mod_name) or os.path.isfile(
            self.start_dir + '/mods/' + mod_name + '.py')):
            try:
                if mfo is None:
                    modular_file_object = import_module(loc + mod_name)
                else:
                    modular_file_object = mfo
                self.modules[mod_name] = modular_file_object
            except ModuleNotFoundError as e:
                self.logger.error(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                self.print(Style.RED(f"module {loc + mod_name} not found is type sensitive {e}"))
                if self.debug or self.args_sto.sysPrint:
                    self.python_module_import_classifier(mod_name, str(e))
                self.debug_rains(e)
                return None
        else:
            self.sprint(f"module {loc + mod_name} is not valid")
            return None
        if hasattr(modular_file_object, "Tools"):
            tools_class = modular_file_object.Tools
        else:
            if hasattr(modular_file_object, "name"):
                tools_class = modular_file_object
                modular_file_object = import_module(loc + mod_name)
            else:
                tools_class = None

        modular_id = None
        instance = modular_file_object
        app_instance_type = "file/application"

        if tools_class is None:
            modular_id = modular_file_object.Name if hasattr(modular_file_object, "Name") else mod_name

        if tools_class is None and modular_id is None:
            modular_id = str(modular_file_object.__name__)
            self.logger.warning(f"Unknown instance loaded {mod_name}")
            return modular_file_object

        if tools_class is not None:
            tools_class = self.save_initialized_module(tools_class, spec)
            modular_id = tools_class.name
            app_instance_type = "functions/class"
        else:
            instance.spec = spec
        # if private:
        #     self.functions[modular_id][f"{spec}_private"] = private

        if not save:
            return instance if tools_class is None else tools_class

        return self.save_instance(instance, modular_id, spec, app_instance_type, tools_class=tools_class)

    def save_instance(self, instance, modular_id, spec='app', instance_type="file/application", tools_class=None):

        if modular_id in self.functions and tools_class is None:
            if self.functions[modular_id].get(f"{spec}_instance", None) is None:
                self.functions[modular_id][f"{spec}_instance"] = instance
                self.functions[modular_id][f"{spec}_instance_type"] = instance_type
            else:
                self.print("Firest instance stays use new spec to get new instance")
                if modular_id in self.functions and self.functions[modular_id].get(f"{spec}_instance", None) is not None:
                    return self.functions[modular_id][f"{spec}_instance"]
                else:
                    raise ImportError(f"Module already known {modular_id} and not avalabel reload using other spec then {spec}")

        elif tools_class is not None:
            if modular_id not in self.functions:
                self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = tools_class
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

            try:
                if not hasattr(tools_class, 'tools'):
                    tools_class.tools = {"Version": tools_class.get_version, 'name': tools_class.name}
                for function_name in list(tools_class.tools.keys()):
                    t_function_name = function_name.lower()
                    if t_function_name != "all" and t_function_name != "name":
                        self.tb(function_name, mod_name=modular_id)(tools_class.tools.get(function_name))
                self.functions[modular_id][f"{spec}_instance_type"] += "/BC"
                if hasattr(tools_class, 'on_exit'):
                    if "on_exit" in self.functions[modular_id]:
                        self.functions[modular_id]["on_exit"].append(tools_class.on_exit)
                    else:
                        self.functions[modular_id]["on_exit"] = [tools_class.on_exit]
            except Exception as e:
                self.logger.error(f"Starting Module {modular_id} compatibility failed with : {e}")
                pass
        elif modular_id not in self.functions and tools_class is None:
            self.functions[modular_id] = {}
            self.functions[modular_id][f"{spec}_instance"] = instance
            self.functions[modular_id][f"{spec}_instance_type"] = instance_type

        else:
            raise ImportError(f"Modular {modular_id} is not a valid mod")
        on_start = self.functions[modular_id].get("on_start")
        if on_start is not None:
            i = 1
            for f in on_start:
                try:
                    f_, e = self.get_function((modular_id, f), state=True, specification=spec)
                    if e == 0:
                        self.logger.info(Style.GREY(f"Running On start {f} {i}/{len(on_start)}"))
                        if asyncio.iscoroutinefunction(f_):
                            self.print(f"Async on start is only in Tool claas supported for {modular_id}.{f}" if tools_class is None else f"initialization starting soon for {modular_id}.{f}")
                            self.run_bg_task_advanced(f_)
                        else:
                            o = f_()
                            if o is not None:
                                self.print(f"Function {modular_id} On start result: {o}")
                    else:
                        self.logger.warning(f"starting function not found {e}")
                except Exception as e:
                    self.logger.debug(Style.YELLOW(
                        Style.Bold(f"modular:{modular_id}.{f} on_start error {i}/{len(on_start)} -> {e}")))
                    self.debug_rains(e)
                finally:
                    i += 1
        return instance if tools_class is None else tools_class

    def save_initialized_module(self, tools_class, spec):
        tools_class.spec = spec
        live_tools_class = tools_class(app=self)
        return live_tools_class

    def mod_online(self, mod_name, installed=False):
        if installed and mod_name not in self.functions:
            self.save_load(mod_name)
        return mod_name in self.functions

    def _get_function(self,
                      name: Enum or None,
                      state: bool = True,
                      specification: str = "app",
                      metadata=False, as_str: tuple or None = None, r=0, **kwargs):

        if as_str is None and isinstance(name, Enum):
            modular_id = str(name.NAME.value)
            function_id = str(name.value)
        elif as_str is None and isinstance(name, list):
            modular_id, function_id = name[0], name[1]
        else:
            modular_id, function_id = as_str

        self.logger.info(f"getting function : {specification}.{modular_id}.{function_id}")

        if modular_id not in self.functions:
            if r == 0:
                self.save_load(modular_id, spec=specification)
                return self.get_function(name=(modular_id, function_id),
                                         state=state,
                                         specification=specification,
                                         metadata=metadata,
                                         r=1)
            self.logger.warning(f"function modular not found {modular_id} 404")
            return "404", 404

        if function_id not in self.functions[modular_id]:
            self.logger.warning(f"function data not found {modular_id}.{function_id} 404")
            return "404", 404

        function_data = self.functions[modular_id][function_id]

        if isinstance(function_data, list):
            print(f"functions {function_id} : {function_data}")
            function_data = self.functions[modular_id][function_data[kwargs.get('i', -1)]]
            print(f"functions {modular_id} : {function_data}")
        function = function_data.get("func")
        params = function_data.get("params")

        state_ = function_data.get("state")
        if state_ is not None and state != state_:
            state = state_

        if function is None:
            self.logger.warning("No function found")
            return "404", 404

        if params is None:
            self.logger.warning("No function (params) found")
            return "404", 301

        if metadata and not state:
            self.logger.info("returning metadata stateless")
            return (function_data, function), 0

        if not state:  # mens a stateless function
            self.logger.info("returning stateless function")
            return function, 0

        instance = self.functions[modular_id].get(f"{specification}_instance")

        # instance_type = self.functions[modular_id].get(f"{specification}_instance_type", "functions/class")

        if params[0] == 'app':
            instance = get_app(from_=f"fuction {specification}.{modular_id}.{function_id}")

        if instance is None and self.alive:
            self.inplace_load_instance(modular_id, spec=specification)
            instance = self.functions[modular_id].get(f"{specification}_instance")

        if instance is None:
            self.logger.warning("No live Instance found")
            return "404", 400

        # if instance_type.endswith("/BC"):  # for backwards compatibility  functions/class/BC old modules
        #     # returning as stateless
        #     # return "422", -1
        #     self.logger.info(
        #         f"returning stateless function, cant find tools class for state handling found {instance_type}")
        #     if metadata:
        #         self.logger.info(f"returning metadata stateless")
        #         return (function_data, function), 0
        #     return function, 0

        self.logger.info("wrapping in higher_order_function")

        self.logger.info(f"returned fuction {specification}.{modular_id}.{function_id}")
        higher_order_function = partial(function, instance)

        if metadata:
            self.logger.info("returning metadata stateful")
            return (function_data, higher_order_function), 0

        self.logger.info("returning stateful function")
        return higher_order_function, 0

    def save_exit(self):
        self.logger.info(f"save exiting saving data to {self.config_fh.file_handler_filename} states of {self.debug=}")
        self.config_fh.add_to_save_file_handler(self.keys["debug"], str(self.debug))

    def init_mod(self, mod_name, spec='app'):
        """
        Initializes a module in a thread-safe manner by submitting the
        asynchronous initialization to the running event loop.
        """
        if '.' in mod_name:
            mod_name = mod_name.split('.')[0]
        self.run_bg_task(self.a_init_mod, mod_name, spec)
        # loop = self.loop_gard()
        # if loop:
        #     # Create a future to get the result from the coroutine
        #     future: Future = asyncio.run_coroutine_threadsafe(
        #         self.a_init_mod(mod_name, spec), loop
        #     )
        #     # Block until the result is available
        #     return future.result()
        # else:
        #     raise ValueError("Event loop is not running")
        #     # return self.loop_gard().run_until_complete(self.a_init_mod(mod_name, spec))

    def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
        """
        Runs a coroutine in the background without blocking the caller.

        This is the primary method for "fire-and-forget" async tasks. It schedules
        the coroutine to run on the application's main event loop.

        Args:
            task: The coroutine function to run.
            *args: Arguments to pass to the coroutine function.
            **kwargs: Keyword arguments to pass to the coroutine function.

        Returns:
            An asyncio.Task object representing the scheduled task, or None if
            the task could not be scheduled.
        """
        if not callable(task):
            self.logger.warning("Task passed to run_bg_task is not callable!")
            return None

        if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
            self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                                f"Use run_bg_task_advanced for synchronous functions.")
            # Fallback to advanced runner for convenience
            self.run_bg_task_advanced(task, *args, **kwargs)
            return None

        try:
            loop = self.loop_gard()
            if not loop.is_running():
                # If the main loop isn't running, we can't create a task on it.
                # This scenario is handled by run_bg_task_advanced.
                self.logger.info("Main event loop not running. Delegating to advanced background runner.")
                return self.run_bg_task_advanced(task, *args, **kwargs)

            # Create the coroutine if it's a function
            coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

            # Create a task on the running event loop
            bg_task = loop.create_task(coro)

            # Add a callback to log exceptions from the background task
            def _log_exception(the_task: asyncio.Task):
                if not the_task.cancelled() and the_task.exception():
                    self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                      exc_info=the_task.exception())

            bg_task.add_done_callback(_log_exception)
            self.bg_tasks.append(bg_task)
            return bg_task

        except Exception as e:
            self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
            return None

    def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
        """
        Runs a task in a separate, dedicated background thread with its own event loop.

        This is ideal for:
        1. Running an async task from a synchronous context.
        2. Launching a long-running, independent operation that should not
           interfere with the main application's event loop.

        Args:
            task: The function to run (can be sync or async).
            *args: Arguments for the task.
            **kwargs: Keyword arguments for the task.

        Returns:
            The threading.Thread object managing the background execution.
        """
        if not callable(task):
            self.logger.warning("Task for run_bg_task_advanced is not callable!")
            return None

        def thread_target():
            # Each thread gets its own event loop.
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

            try:
                # Prepare the coroutine we need to run
                if asyncio.iscoroutinefunction(task):
                    coro = task(*args, **kwargs)
                elif asyncio.iscoroutine(task):
                    # It's already a coroutine object
                    coro = task
                else:
                    # It's a synchronous function, run it in an executor
                    # to avoid blocking the new event loop.
                    coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

                # Run the coroutine to completion
                result = loop.run_until_complete(coro)
                self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
                if result is not None:
                    self.logger.debug(f"Task result: {str(result)[:100]}")

            except Exception as e:
                self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                                  exc_info=e)
            finally:
                # Cleanly shut down the event loop in this thread.
                try:
                    all_tasks = asyncio.all_tasks(loop=loop)
                    if all_tasks:
                        for t in all_tasks:
                            t.cancel()
                        loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
                finally:
                    loop.close()
                    asyncio.set_event_loop(None)

        # Create, start, and return the thread.
        # It's a daemon thread so it won't prevent the main app from exiting.
        t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
        self.bg_tasks.append(t)
        t.start()
        return t

    # Helper method to wait for background tasks to complete (optional)
    def wait_for_bg_tasks(self, timeout=None):
        """
        Wait for all background tasks to complete.

        Args:
            timeout: Maximum time to wait (in seconds) for all tasks to complete.
                     None means wait indefinitely.

        Returns:
            bool: True if all tasks completed, False if timeout occurred
        """
        active_tasks = [t for t in self.bg_tasks if t.is_alive()]

        for task in active_tasks:
            task.join(timeout=timeout)
            if task.is_alive():
                return False

        return True

    def __call__(self, *args, **kwargs):
        return self.run(*args, **kwargs)

    def run(self, *args, request=None, running_function_coro=None, **kwargs):
        """
        Run a function with support for SSE streaming in both
        threaded and non-threaded contexts.
        """
        if running_function_coro is None:
            mn, fn = args[0]
            if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
                kwargs["request"] = RequestData.from_dict(request)
                if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                    kwargs["request"].data = kwargs["request"].body = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                           []):
                    kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                    del kwargs['form_data']
            else:
                params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
                # auto pars data and form_data to kwargs by key
                do = False
                data = {}
                if 'data' in kwargs and 'data' not in params:
                    do = True
                    data = kwargs['data']
                    del kwargs['data']
                if 'form_data' in kwargs and 'form_data' not in params:
                    do = True
                    data = kwargs['form_data']
                    del kwargs['form_data']
                if do:
                    for k in params:
                        if k in data:
                            kwargs[k] = data[k]
                            del data[k]

        # Create the coroutine
        coro = running_function_coro or self.a_run_any(*args, **kwargs)

        # Get or create an event loop
        try:
            loop = asyncio.get_event_loop()
            is_running = loop.is_running()
        except RuntimeError:
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)
            is_running = False

        # If the loop is already running, run in a separate thread
        if is_running:
            # Create thread pool executor as needed
            if not hasattr(self.__class__, '_executor'):
                self.__class__._executor = ThreadPoolExecutor(max_workers=4)

            def run_in_new_thread():
                # Set up a new loop in this thread
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)

                try:
                    # Run the coroutine
                    return new_loop.run_until_complete(coro)
                finally:
                    new_loop.close()

            # Run in thread and get result
            thread_result = self.__class__._executor.submit(run_in_new_thread).result()

            # Handle streaming results from thread
            if isinstance(thread_result, dict) and thread_result.get("is_stream"):
                # Create a new SSE stream in the main thread
                async def stream_from_function():
                    # Re-run the function with direct async access
                    stream_result = await self.a_run_any(*args, **kwargs)

                    if (isinstance(stream_result, Result) and
                        getattr(stream_result.result, 'data_type', None) == "stream"):
                        # Get and forward data from the original generator
                        original_gen = stream_result.result.data.get("generator")
                        if inspect.isasyncgen(original_gen):
                            async for item in original_gen:
                                yield item

                # Return a new streaming Result
                return Result.stream(
                    stream_generator=stream_from_function(),
                    headers=thread_result.get("headers", {})
                )

            result = thread_result
        else:
            # Direct execution when loop is not running
            result = loop.run_until_complete(coro)

        # Process the final result
        if isinstance(result, Result):
            if 'debug' in self.id:
                result.print()
            if getattr(result.result, 'data_type', None) == "stream":
                return result
            return result.to_api_result().model_dump(mode='json')

        return result

    def loop_gard(self):
        if self.loop is None:
            self._start_event_loop()
            self.loop = asyncio.get_event_loop()
        if self.loop.is_closed():
            self.loop = asyncio.get_event_loop()
        return self.loop

    async def a_init_mod(self, mod_name, spec='app'):
        mod = self.save_load(mod_name, spec=spec)
        if hasattr(mod, "__initobj") and not mod.async_initialized:
            await mod
        return mod


    def load_mod(self, mod_name: str, mlm='I', **kwargs):

        action_list_helper = ['I (inplace load dill on error python)',
                              # 'C (coppy py file to runtime dir)',
                              # 'S (save py file to dill)',
                              # 'CS (coppy and save py file)',
                              # 'D (development mode, inplace load py file)'
                              ]
        action_list = {"I": lambda: self.inplace_load_instance(mod_name, **kwargs),
                       "C": lambda: self._copy_load(mod_name, **kwargs)
                       }

        try:
            if mlm in action_list:

                return action_list.get(mlm)()
            else:
                self.logger.critical(
                    f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
                raise ValueError(f"config mlm must be {' or '.join(action_list_helper)} is {mlm=}")
        except ValueError as e:
            self.logger.warning(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except ImportError as e:
            self.logger.error(Style.YELLOW(f"Error Loading Module '{mod_name}', with error :{e}"))
            self.debug_rains(e)
        except Exception as e:
            self.logger.critical(Style.RED(f"Error Loading Module '{mod_name}', with critical error :{e}"))
            print(Style.RED(f"Error Loading Module '{mod_name}'"))
            self.debug_rains(e)

        return Result.default_internal_error(info="info's in logs.")

    async def load_external_mods(self):
        for mod_path in os.getenv("EXTERNAL_PATH_RUNNABLE", '').split(','):
            if mod_path:
                await self.load_all_mods_in_file(mod_path)

    async def load_all_mods_in_file(self, working_dir="mods"):
        print(f"LOADING ALL MODS FROM FOLDER : {working_dir}")
        t0 = time.perf_counter()
        # Get the list of all modules
        module_list = self.get_all_mods(working_dir)
        open_modules = self.functions.keys()
        start_len = len(open_modules)

        for om in open_modules:
            if om in module_list:
                module_list.remove(om)

        tasks: set[Task] = set()

        _ = {tasks.add(asyncio.create_task(asyncio.to_thread(self.save_load, mod, 'app'))) for mod in module_list}
        for t in asyncio.as_completed(tasks):
            try:
                result = await t
                if hasattr(result, 'Name'):
                    self.print('Opened :', result.Name)
                elif hasattr(result, 'name'):
                    if hasattr(result, 'async_initialized'):
                        if not result.async_initialized:
                            async def _():
                                try:
                                    if asyncio.iscoroutine(result):
                                        await result
                                    if hasattr(result, 'Name'):
                                        self.print('Opened :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Opened :', result.name)
                                except Exception as e:
                                    self.debug_rains(e)
                                    if hasattr(result, 'Name'):
                                        self.print('Error opening :', result.Name)
                                    elif hasattr(result, 'name'):
                                        self.print('Error opening :', result.name)
                            asyncio.create_task(_())
                        else:
                            self.print('Opened :', result.name)
                else:
                    if result:
                        self.print('Opened :', result)
            except Exception as e:
                self.logger.error(Style.RED(f"An Error occurred while opening all modules error: {str(e)}"))
                self.debug_rains(e)
        opened = len(self.functions.keys()) - start_len

        self.logger.info(f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s")
        return f"Opened {opened} modules in {time.perf_counter() - t0:.2f}s"

    def get_all_mods(self, working_dir="mods", path_to="./runtime", use_wd=True):
        self.logger.info(f"collating all mods in working directory {working_dir}")

        pr = "_dev" if self.dev_modi else ""
        if working_dir == "mods" and use_wd:
            working_dir = f"{self.start_dir}/mods{pr}"
        elif use_wd:
            pass
        else:
            w_dir = self.id.replace(".", "_")
            working_dir = f"{path_to}/{w_dir}/mod_lib{pr}/"
        res = os.listdir(working_dir)

        self.logger.info(f"found : {len(res)} files")

        def do_helper(_mod):
            if "mainTool" in _mod:
                return False
            # if not _mod.endswith(".py"):
            #     return False
            if _mod.startswith("__"):
                return False
            if _mod.startswith("."):
                return False
            return not _mod.startswith("test_")

        def r_endings(word: str):
            if word.endswith(".py"):
                return word[:-3]
            return word

        mods_list = list(map(r_endings, filter(do_helper, res)))

        self.logger.info(f"found : {len(mods_list)} Modules")
        return mods_list

    def remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            self.remove_mod(mod, delete=delete)

    def remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return

        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    self.exit_tasks.append(instance.on_exit)
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1

        for j, f in enumerate(on_exit):
            try:
                f_, e = self.get_function((mod_name, f), state=True, specification=spec, i=j)
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        self.exit_tasks.append(f_)
                        o = None
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))

                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    async def a_remove_all_modules(self, delete=False):
        for mod in list(self.functions.keys()):
            self.logger.info(f"closing: {mod}")
            await self.a_remove_mod(mod, delete=delete)

    async def a_remove_mod(self, mod_name, spec='app', delete=True):
        if mod_name not in self.functions:
            self.logger.info(f"mod not active {mod_name}")
            return
        on_exit = self.functions[mod_name].get("on_exit")
        self.logger.info(f"closing: {on_exit}")
        def helper():
            if f"{spec}_instance" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance"]
            if f"{spec}_instance_type" in self.functions[mod_name]:
                del self.functions[mod_name][f"{spec}_instance_type"]

        if on_exit is None and self.functions[mod_name].get(f"{spec}_instance_type", "").endswith("/BC"):
            instance = self.functions[mod_name].get(f"{spec}_instance", None)
            if instance is not None and hasattr(instance, 'on_exit'):
                if asyncio.iscoroutinefunction(instance.on_exit):
                    await instance.on_exit()
                else:
                    instance.on_exit()

        if on_exit is None and delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]
            return
        if on_exit is None:
            helper()
            return

        i = 1
        for f in on_exit:
            try:
                e = 1
                if isinstance(f, str):
                    f_, e = self.get_function((mod_name, f), state=True, specification=spec)
                elif isinstance(f, Callable):
                    f_, e, f  = f, 0, f.__name__
                if e == 0:
                    self.logger.info(Style.GREY(f"Running On exit {f} {i}/{len(on_exit)}"))
                    if asyncio.iscoroutinefunction(f_):
                        o = await f_()
                    else:
                        o = f_()
                    if o is not None:
                        self.print(f"Function On Exit result: {o}")
                else:
                    self.logger.warning("closing function not found")
            except Exception as e:
                self.logger.debug(
                    Style.YELLOW(Style.Bold(f"modular:{mod_name}.{f} on_exit error {i}/{len(on_exit)} -> {e}")))
                self.debug_rains(e)
            finally:
                i += 1

        helper()

        if delete:
            self.functions[mod_name] = {}
            del self.functions[mod_name]

    def exit(self, remove_all=True):
        if not self.alive:
            return
        if self.args_sto.debug:
            self.hide_console()
        self.disconnect()
        if remove_all:
            self.remove_all_modules()
        self.logger.info("Exiting ToolBox interface")
        self.alive = False
        self.called_exit = True, time.time()
        self.save_exit()
        if hasattr(self, 'root_blob_storage') and self.root_blob_storage:
            self.root_blob_storage.exit()
        try:
            self.config_fh.save_file_handler()
        except SystemExit:
            print("If u ar testing this is fine else ...")

        if hasattr(self, 'daemon_app'):
            import threading

            for thread in threading.enumerate()[::-1]:
                if thread.name == "MainThread":
                    continue
                try:
                    with Spinner(f"closing Thread {thread.name:^50}|", symbols="s", count_down=True,
                                 time_in_s=0.751 if not self.debug else 0.6):
                        thread.join(timeout=0.751 if not self.debug else 0.6)
                except TimeoutError as e:
                    self.logger.error(f"Timeout error on exit {thread.name} {str(e)}")
                    print(str(e), f"Timeout {thread.name}")
                except KeyboardInterrupt:
                    print("Unsave Exit")
                    break
        if hasattr(self, 'loop') and self.loop is not None:
            with Spinner("closing Event loop:", symbols="+"):
                self.loop.stop()

    async def a_exit(self):

        import inspect
        self.sprint(f"exit requested from: {inspect.stack()[1].filename}::{inspect.stack()[1].lineno} function: {inspect.stack()[1].function}")

        # Cleanup session before removing modules
        try:
            if hasattr(self, 'session') and self.session is not None:
                await self.session.cleanup()
        except Exception as e:
            self.logger.debug(f"Session cleanup error (ignored): {e}")

        await self.a_remove_all_modules(delete=True)
        results = await asyncio.gather(
            *[asyncio.create_task(f()) for f in self.exit_tasks if asyncio.iscoroutinefunction(f)])
        for result in results:
            self.print(f"Function On Exit result: {result}")
        self.exit(remove_all=False)

    def save_load(self, modname, spec='app'):
        self.logger.debug(f"Save load module {modname}")
        if not modname:
            self.logger.warning("no filename specified")
            return False
        try:
            return self.load_mod(modname, spec=spec)
        except ModuleNotFoundError as e:
            self.logger.error(Style.RED(f"Module {modname} not found"))
            self.debug_rains(e)

        return False

    def get_function(self, name: Enum or tuple, **kwargs):
        """
        Kwargs for _get_function
            metadata:: return the registered function dictionary
                stateless: (function_data, None), 0
                stateful: (function_data, higher_order_function), 0
            state::boolean
                specification::str default app
        """
        if isinstance(name, tuple):
            return self._get_function(None, as_str=name, **kwargs)
        else:
            return self._get_function(name, **kwargs)

    async def a_run_function(self, mod_function_name: Enum or tuple,
                             tb_run_function_with_state=True,
                             tb_run_with_specification='app',
                             args_=None,
                             kwargs_=None,
                             *args,
                             **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if tb_run_with_specification == 'ws_internal':
            modular_name = modular_name.split('/')[0]
            if not self.mod_online(modular_name, installed=True):
                self.get_mod(modular_name)
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        await handler_func(self, **kwargs)
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 404:
            mod = self.get_mod(modular_name)
            if hasattr(mod, "async_initialized") and not mod.async_initialized:
                await mod
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 404:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == 300:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            return await self.a_fuction_runner(function, function_data, args, kwargs, t0)
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)


    def run_function(self, mod_function_name: Enum or tuple,
                     tb_run_function_with_state=True,
                     tb_run_with_specification='app',
                     args_=None,
                     kwargs_=None,
                     *args,
                     **kwargs) -> Result:

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_
        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value
        else:
            raise TypeError("Unknown function type")

        if not self.mod_online(modular_name, installed=True):
            self.get_mod(modular_name)

        if tb_run_with_specification == 'ws_internal':
            handler_id, event_name = mod_function_name
            if handler_id in self.websocket_handlers and event_name in self.websocket_handlers[handler_id]:
                handler_func = self.websocket_handlers[handler_id][event_name]
                try:
                    # Führe den asynchronen Handler aus
                    if inspect.iscoroutinefunction(handler_func):
                        return self.loop.run_until_complete(handler_func(self, **kwargs))
                    else:
                        handler_func(self, **kwargs)  # Für synchrone Handler
                    return Result.ok(info=f"WS handler '{event_name}' executed.")
                except Exception as e:
                    self.logger.error(f"Error in WebSocket handler '{handler_id}/{event_name}': {e}", exc_info=True)
                    return Result.default_internal_error(info=str(e))
            else:
                # Kein Handler registriert, aber das ist kein Fehler (z.B. on_connect ist optional)
                return Result.ok(info=f"No WS handler for '{event_name}'.")

        function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                      metadata=True, specification=tb_run_with_specification)
        self.logger.info(f"Received fuction : {mod_function_name}, with execode: {error_code}")
        if error_code == 1 or error_code == 3 or error_code == 400:
            self.get_mod(modular_name)
            function_data, error_code = self.get_function(mod_function_name, state=tb_run_function_with_state,
                                                          metadata=True, specification=tb_run_with_specification)

        if error_code == 2:
            self.logger.warning(Style.RED("Function Not Found"))
            return (Result.default_user_error(interface=self.interface_type,
                                              exec_code=404,
                                              info="function not found function is not decorated").
                    set_origin(mod_function_name))

        if error_code == -1:
            return Result.default_internal_error(interface=self.interface_type,
                                                 info=f"module {modular_name}"
                                                      f" has no state (instance)").set_origin(mod_function_name)

        if error_code != 0:
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=error_code,
                                                 info=f"Internal error"
                                                      f" {modular_name}."
                                                      f"{function_name}").set_origin(mod_function_name)

        if not tb_run_function_with_state:
            function_data, _ = function_data
            function = function_data.get('func')
        else:
            function_data, function = function_data

        if not function:
            self.logger.warning(Style.RED(f"Function {function_name} not found"))
            return Result.default_internal_error(interface=self.interface_type,
                                                 exec_code=404,
                                                 info="function not found function").set_origin(mod_function_name)

        self.logger.info("Profiling function")
        t0 = time.perf_counter()
        if asyncio.iscoroutinefunction(function):
            try:
                return asyncio.run(self.a_fuction_runner(function, function_data, args, kwargs, t0))
            except RuntimeError:
                try:
                    return self.loop.run_until_complete(self.a_fuction_runner(function, function_data, args, kwargs, t0))
                except RuntimeError:
                    pass
            raise ValueError(f"Fuction {function_name} is Async use a_run_any")
        else:
            return self.fuction_runner(function, function_data, args, kwargs, t0)

    def run_a_from_sync(self, function, *args, **kwargs):
        # Initialize self.loop if not already set.
        if self.loop is None:
            try:
                self.loop = asyncio.get_running_loop()
            except RuntimeError:
                self.loop = asyncio.new_event_loop()

        # If the loop is running, offload the coroutine to a new thread.
        if self.loop.is_running():
            result_future = Future()

            def run_in_new_loop():
                new_loop = asyncio.new_event_loop()
                asyncio.set_event_loop(new_loop)
                try:
                    result = new_loop.run_until_complete(function(*args, **kwargs))
                    result_future.set_result(result)
                except Exception as e:
                    result_future.set_exception(e)
                finally:
                    new_loop.close()

            thread = threading.Thread(target=run_in_new_loop)
            thread.start()
            thread.join()  # Block until the thread completes.
            return result_future.result()
        else:
            # If the loop is not running, schedule and run the coroutine directly.
            future = self.loop.create_task(function(*args, **kwargs))
            return self.loop.run_until_complete(future)

    def fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = function()
            elif len(parameters) == len(args) + if_self_state:
                res = function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = function(**kwargs)
            else:
                res = function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)
            self.print(f"! Function ERROR: in {modular_name}.{function_name} ")



        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def a_fuction_runner(self, function, function_data: dict, args: list, kwargs: dict, t0=.0):

        parameters = function_data.get('params')
        modular_name = function_data.get('module_name')
        function_name = function_data.get('func_name')
        row = function_data.get('row')
        mod_function_name = f"{modular_name}.{function_name}"

        if_self_state = 1 if 'self' in parameters else 0

        try:
            if len(parameters) == 0:
                res = await function()
            elif len(parameters) == len(args) + if_self_state:
                res = await function(*args)
            elif len(parameters) == len(kwargs.keys()) + if_self_state:
                res = await function(**kwargs)
            else:
                res = await function(*args, **kwargs)
            self.logger.info(f"Execution done in {time.perf_counter()-t0:.4f}")
            if isinstance(res, Result):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.set_origin(mod_function_name)
            elif isinstance(res, ApiResult):
                formatted_result = res
                if formatted_result.origin is None:
                    formatted_result.as_result().set_origin(mod_function_name).to_api_result()
            elif row:
                formatted_result = res
            else:
                # Wrap the result in a Result object
                formatted_result = Result.ok(
                    interface=self.interface_type,
                    data_info="Auto generated result",
                    data=res,
                    info="Function executed successfully"
                ).set_origin(mod_function_name)
            if not row:
                self.logger.info(
                    f"Function Exec code: {formatted_result.info.exec_code} Info's: {formatted_result.info.help_text}")
            else:
                self.logger.info(
                    f"Function Exec data: {formatted_result}")
        except Exception as e:
            self.logger.error(
                Style.YELLOW(Style.Bold(
                    f"! Function ERROR: in {modular_name}.{function_name}")))
            # Wrap the exception in a Result object
            formatted_result = Result.default_internal_error(info=str(e)).set_origin(mod_function_name)
            # res = formatted_result
            self.logger.error(
                f"Function {modular_name}.{function_name}"
                f" executed wit an error {str(e)}, {type(e)}")
            self.debug_rains(e)

        else:
            self.print_ok()

            self.logger.info(
                f"Function {modular_name}.{function_name}"
                f" executed successfully")

        return formatted_result

    async def run_http(self, mod_function_name: Enum or str or tuple, function_name=None,
                       args_=None,
                       kwargs_=None, method="GET",
                       *args, **kwargs):
        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        modular_name = mod_function_name
        function_name = function_name

        if isinstance(mod_function_name, str) and isinstance(function_name, str):
            mod_function_name = (mod_function_name, function_name)

        if isinstance(mod_function_name, tuple):
            modular_name, function_name = mod_function_name
        elif isinstance(mod_function_name, list):
            modular_name, function_name = mod_function_name[0], mod_function_name[1]
        elif isinstance(mod_function_name, Enum):
            modular_name, function_name = mod_function_name.__class__.NAME.value, mod_function_name.value

        self.logger.info(f"getting function : {modular_name}.{function_name} from http {self.session.base}")
        r = await self.session.fetch(f"/api/{modular_name}/{function_name}{'?' + args_ if args_ is not None else ''}",
                                     data=kwargs, method=method)
        try:
            if not r:
                print("§ Session server Offline!", self.session.base)
                return Result.default_internal_error(info="Session fetch failed").as_dict()

            content_type = r.headers.get('Content-Type', '').lower()

            if 'application/json' in content_type:
                try:
                    return r.json()
                except Exception as e:
                    print(f"⚠ JSON decode error: {e}")
                    # Fallback to text if JSON decoding fails
                    text = r.text
            else:
                text = r.text

            if isinstance(text, Callable):
                if asyncio.iscoroutinefunction(text):
                    text = await text()
                else:
                    text = text()

            # Attempt YAML
            if 'yaml' in content_type or text.strip().startswith('---'):
                try:
                    import yaml
                    return yaml.safe_load(text)
                except Exception as e:
                    print(f"⚠ YAML decode error: {e}")

            # Attempt XML
            if 'xml' in content_type or text.strip().startswith('<?xml'):
                try:
                    import xmltodict
                    return xmltodict.parse(text)
                except Exception as e:
                    print(f"⚠ XML decode error: {e}")

            # Fallback: return plain text
            return Result.default_internal_error(data={'raw_text': text, 'content_type': content_type}).as_dict()

        except Exception as e:
            print("❌ Fatal error during API call:", e)
            self.debug_rains(e)
            return Result.default_internal_error(str(e)).as_dict()

    def run_local(self, *args, **kwargs):
        return self.run_any(*args, **kwargs)

    async def a_run_local(self, *args, **kwargs):
        return await self.a_run_any(*args, **kwargs)

    def run_any(self, mod_function_name: Enum or str or tuple, backwords_compability_variabel_string_holder=None,
                get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                kwargs_=None,
                *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = self.run_function(mod_function_name,
                                        tb_run_function_with_state=tb_run_function_with_state,
                                        tb_run_with_specification=tb_run_with_specification,
                                        args_=args, kwargs_=kwargs).as_result()
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.log(show_data=False)

        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res

    async def a_run_any(self, mod_function_name: Enum or str or tuple,
                        backwords_compability_variabel_string_holder=None,
                        get_results=False, tb_run_function_with_state=True, tb_run_with_specification='app', args_=None,
                        kwargs_=None,
                        *args, **kwargs):

        # if self.debug:
        #     self.logger.info(f'Called from: {getouterframes(currentframe(), 2)}')

        if kwargs_ is not None and not kwargs:
            kwargs = kwargs_
        if args_ is not None and not args:
            args = args_

        if isinstance(mod_function_name, str) and backwords_compability_variabel_string_holder is None:
            backwords_compability_variabel_string_holder = mod_function_name.split('.')[-1]
            mod_function_name = mod_function_name.replace(f".{backwords_compability_variabel_string_holder}", "")

        if isinstance(mod_function_name, str) and isinstance(backwords_compability_variabel_string_holder, str):
            mod_function_name = (mod_function_name, backwords_compability_variabel_string_holder)

        res: Result = await self.a_run_function(mod_function_name,
                                                tb_run_function_with_state=tb_run_function_with_state,
                                                tb_run_with_specification=tb_run_with_specification,
                                                args_=args, kwargs_=kwargs)
        if isinstance(res, ApiResult):
            res = res.as_result()

        if isinstance(res, Result) and res.bg_task is not None:
            self.run_bg_task(res.bg_task)

        if self.debug:
            res.print()
            res.log(show_data=False) if isinstance(res, Result) else self.logger.debug(res)
        if not get_results and isinstance(res, Result):
            return res.get()

        if get_results and not isinstance(res, Result):
            return Result.ok(data=res)

        return res


    def web_context(self):
        if self._web_context is None:
            try:
                self._web_context = open("./dist/helper.html", encoding="utf-8").read()
            except Exception as e:
                self.logger.error(f"Could not load web context: {e}")
                self._web_context = "<div><h1>Web Context not found</h1></div>"
        return self._web_context

    def get_mod(self, name, spec='app') -> ModuleType or MainToolType:
        if spec != "app":
            self.print(f"Getting Module {name} spec: {spec}")
        if name not in self.functions:
            mod = self.save_load(name, spec=spec)
            if mod is False or (isinstance(mod, Result) and mod.is_error()):
                self.logger.warning(f"Could not find {name} in {list(self.functions.keys())}")
                raise ValueError(f"Could not find {name} in {list(self.functions.keys())} pleas install the module, or its posibly broken use --debug for infos")
        # private = self.functions[name].get(f"{spec}_private")
        # if private is not None:
        #     if private and spec != 'app':
        #         raise ValueError("Module is private")
        if name not in self.functions:
            self.logger.warning(f"Module '{name}' is not found")
            return None
        instance = self.functions[name].get(f"{spec}_instance")
        if instance is None:
            return self.load_mod(name, spec=spec)
        return self.functions[name].get(f"{spec}_instance")

    def print(self, text="", *args, **kwargs):
        # self.logger.info(f"Output : {text}")
        if 'live' in self.id:
            return

        flush = kwargs.pop('flush', True)
        if self.sprint(None):
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        print(text, *args, **kwargs, flush=flush)

    def sprint(self, text="", show_system=True, *args, **kwargs):
        if text is None:
            return True
        if 'live' in self.id:
            return
        flush = kwargs.pop('flush', True)
        # self.logger.info(f"Output : {text}")
        if show_system:
            print(Style.CYAN(f"System${self.id}:"), end=" ", flush=flush)
        if isinstance(text, str) and kwargs == {} and text:
            stram_print(text + ' '.join(args))
            print()
        else:
            print(text, *args, **kwargs, flush=flush)

    # ----------------------------------------------------------------
    # Decorators for the toolbox

    def reload_mod(self, mod_name, spec='app', is_file=True, loc="toolboxv2.mods."):
        self.remove_mod(mod_name, delete=True)
        if mod_name not in self.modules:
            self.logger.warning(f"Module '{mod_name}' is not found")
            return
        if hasattr(self.modules[mod_name], 'reload_save') and self.modules[mod_name].reload_save:
            def reexecute_module_code(x):
                return x
        else:
            def reexecute_module_code(module_name):
                if isinstance(module_name, str):
                    module = import_module(module_name)
                else:
                    module = module_name
                # Get the source code of the module
                try:
                    source = inspect.getsource(module)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    return module
                # Compile the source code
                try:
                    code = compile(source, module.__file__, 'exec')
                    # Execute the code in the module's namespace
                    exec(code, module.__dict__)
                except Exception:
                    # print(f"No source for {str(module_name).split('from')[0]}: {e}")
                    pass
                return module

        if not is_file:
            mods = self.get_all_mods("./mods/" + mod_name)
            def recursive_reload(package_name):
                package = import_module(package_name)

                # First, reload all submodules
                if hasattr(package, '__path__'):
                    for _finder, name, _ispkg in pkgutil.walk_packages(package.__path__, package.__name__ + "."):
                        try:
                            mod = import_module(name)
                            reexecute_module_code(mod)
                            reload(mod)
                        except Exception as e:
                            print(f"Error reloading module {name}: {e}")
                            break

                # Finally, reload the package itself
                reexecute_module_code(package)
                reload(package)

            for mod in mods:
                if mod.endswith(".txt") or mod.endswith(".yaml"):
                    continue
                try:
                    recursive_reload(loc + mod_name + '.' + mod)
                    self.print(f"Reloaded {mod_name}.{mod}")
                except ImportError:
                    self.print(f"Could not load {mod_name}.{mod}")
        reexecute_module_code(self.modules[mod_name])
        if mod_name in self.functions:
            if "on_exit" in self.functions[mod_name]:
                self.functions[mod_name]["on_exit"] = []
            if "on_start" in self.functions[mod_name]:
                self.functions[mod_name]["on_start"] = []
        self.inplace_load_instance(mod_name, spec=spec, mfo=reload(self.modules[mod_name]) if mod_name in self.modules else None)

    def watch_mod(self, mod_name, spec='app', loc="toolboxv2.mods.", use_thread=True, path_name=None, on_reload=None):
        if path_name is None:
            path_name = mod_name
        is_file = os.path.isfile(self.start_dir + '/mods/' + path_name + '.py')
        import watchfiles
        def helper():
            paths = f'mods/{path_name}' + ('.py' if is_file else '')
            self.logger.info(f'Watching Path: {paths}')
            try:
                for changes in watchfiles.watch(paths):
                    if not changes:
                        continue
                    self.reload_mod(mod_name, spec, is_file, loc)
                    if on_reload:
                        on_reload()
            except FileNotFoundError:
                self.logger.warning(f"Path {paths} not found")

        if not use_thread:
            helper()
        else:
            threading.Thread(target=helper, daemon=True).start()

    def _register_function(self, module_name, func_name, data):
        if module_name not in self.functions:
            self.functions[module_name] = {}
        if func_name in self.functions[module_name]:
            self.print(f"Overriding function {func_name} from {module_name}", end="\r")
            self.functions[module_name][func_name] = data
        else:
            self.functions[module_name][func_name] = data

    def _create_decorator(self, type_: str,
                          name: str = "",
                          mod_name: str = "",
                          level: int = -1,
                          restrict_in_virtual_mode: bool = False,
                          api: bool = False,
                          helper: str = "",
                          version: str or None = None,
                          initial: bool=False,
                          exit_f: bool=False,
                          test: bool=True,
                          samples:list[dict[str, Any]] | None=None,
                          state:bool | None=None,
                          pre_compute:Callable | None=None,
                          post_compute:Callable[[], Result] | None=None,
                          api_methods:list[str] | None=None,
                          memory_cache: bool=False,
                          file_cache: bool=False,
                          request_as_kwarg: bool=False,
                          row: bool=False,
                          memory_cache_max_size:int=100,
                          memory_cache_ttl:int=300,
                          websocket_handler: str | None = None,
                          ):

        if isinstance(type_, Enum):
            type_ = type_.value

        if memory_cache and file_cache:
            raise ValueError("Don't use both cash at the same time for the same fuction")

        use_cache = memory_cache or file_cache
        cache = {}
        if file_cache:
            cache = FileCache(folder=self.data_dir + f'\\cache\\{mod_name}\\',
                              filename=self.data_dir + f'\\cache\\{mod_name}\\{name}cache.db')
        if memory_cache:
            cache = MemoryCache(maxsize=memory_cache_max_size, ttl=memory_cache_ttl)

        version = self.version if version is None else self.version + ':' + version

        def a_additional_process(func):

            async def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = await pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = await func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = await post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            async def wrapper(*args, **kwargs):

                if not use_cache:
                    return await executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = await executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def additional_process(func):

            def executor(*args, **kwargs):

                if pre_compute is not None:
                    args, kwargs = pre_compute(*args, **kwargs)
                if asyncio.iscoroutinefunction(func):
                    result = func(*args, **kwargs)
                else:
                    result = func(*args, **kwargs)
                if post_compute is not None:
                    result = post_compute(result)
                if row:
                    return result
                if not isinstance(result, Result):
                    result = Result.ok(data=result)
                if result.origin is None:
                    result.set_origin((mod_name if mod_name else func.__module__.split('.')[-1]
                                       , name if name else func.__name__
                                       , type_))
                if result.result.data_to == ToolBoxInterfaces.native.name:
                    result.result.data_to = ToolBoxInterfaces.remote if api else ToolBoxInterfaces.native
                # Wenden Sie die to_api_result Methode auf das Ergebnis an, falls verfügbar
                if api and hasattr(result, 'to_api_result'):
                    return result.to_api_result()
                return result

            @wraps(func)
            def wrapper(*args, **kwargs):

                if not use_cache:
                    return executor(*args, **kwargs)

                try:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{str(args)},{str(kwargs.items())}")
                except ValueError:
                    cache_key = (f"{mod_name if mod_name else func.__module__.split('.')[-1]}"
                                 f"-{func.__name__}-{bytes(args)},{str(kwargs.items())}")

                result = cache.get(cache_key)
                if result is not None:
                    return result

                result = executor(*args, **kwargs)

                cache.set(cache_key, result)

                return result

            return wrapper

        def decorator(func):
            sig = signature(func)
            params = list(sig.parameters)
            module_name = mod_name if mod_name else func.__module__.split('.')[-1]
            func_name = name if name else func.__name__
            if func_name == 'on_start':
                func_name = 'on_startup'
            if func_name == 'on_exit':
                func_name = 'on_close'
            if api or pre_compute is not None or post_compute is not None or memory_cache or file_cache:
                if asyncio.iscoroutinefunction(func):
                    func = a_additional_process(func)
                else:
                    func = additional_process(func)
            if api and str(sig.return_annotation) == 'Result':
                raise ValueError(f"Fuction {module_name}.{func_name} registered as "
                                 f"Api fuction but uses {str(sig.return_annotation)}\n"
                                 f"Please change the sig from ..)-> Result to ..)-> ApiResult")
            data = {
                "type": type_,
                "module_name": module_name,
                "func_name": func_name,
                "level": level,
                "restrict_in_virtual_mode": restrict_in_virtual_mode,
                "func": func,
                "api": api,
                "helper": helper,
                "version": version,
                "initial": initial,
                "exit_f": exit_f,
                "api_methods": api_methods if api_methods is not None else ["AUTO"],
                "__module__": func.__module__,
                "signature": sig,
                "params": params,
                "row": row,
                "state": (
                    False if len(params) == 0 else params[0] in ['self', 'state', 'app']) if state is None else state,
                "do_test": test,
                "samples": samples,
                "request_as_kwarg": request_as_kwarg,

            }

            if websocket_handler:
                # Die dekorierte Funktion sollte ein Dict mit den Handlern zurückgeben
                try:
                    handler_config = func(self)  # Rufe die Funktion auf, um die Konfiguration zu erhalten
                    if not isinstance(handler_config, dict):
                        raise TypeError(
                            f"WebSocket handler function '{func.__name__}' must return a dictionary of handlers.")

                    # Handler-Identifikator, z.B. "ChatModule/room_chat"
                    handler_id = f"{module_name}/{websocket_handler}"
                    self.websocket_handlers[handler_id] = {}

                    for event_name, handler_func in handler_config.items():
                        if event_name in ["on_connect", "on_message", "on_disconnect"] and callable(handler_func):
                            self.websocket_handlers[handler_id][event_name] = handler_func
                        else:
                            self.logger.warning(f"Invalid WebSocket handler event '{event_name}' in '{handler_id}'.")

                    self.logger.info(f"Registered WebSocket handlers for '{handler_id}'.")

                except Exception as e:
                    self.logger.error(f"Failed to register WebSocket handlers for '{func.__name__}': {e}",
                                      exc_info=True)
            else:
                self._register_function(module_name, func_name, data)

            if exit_f:
                if "on_exit" not in self.functions[module_name]:
                    self.functions[module_name]["on_exit"] = []
                self.functions[module_name]["on_exit"].append(func_name)
            if initial:
                if "on_start" not in self.functions[module_name]:
                    self.functions[module_name]["on_start"] = []
                self.functions[module_name]["on_start"].append(func_name)

            return func

        decorator.tb_init = True

        return decorator

    def export(self, *args, **kwargs):
        return self.tb(*args, **kwargs)

    def tb(self, name=None,
           mod_name: str = "",
           helper: str = "",
           version: str | None = None,
           test: bool = True,
           restrict_in_virtual_mode: bool = False,
           api: bool = False,
           initial: bool = False,
           exit_f: bool = False,
           test_only: bool = False,
           memory_cache: bool = False,
           file_cache: bool = False,
           request_as_kwarg: bool = False,
           row: bool = False,
           state: bool | None = None,
           level: int = -1,
           memory_cache_max_size: int = 100,
           memory_cache_ttl: int = 300,
           samples: list or dict or None = None,
           interface: ToolBoxInterfaces or None or str = None,
           pre_compute=None,
           post_compute=None,
           api_methods=None,
           websocket_handler: str | None = None,
           ):
        """
    A decorator for registering and configuring functions within a module.

    This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

    Args:
        name (str, optional): The name to register the function under. Defaults to the function's own name.
        mod_name (str, optional): The name of the module the function belongs to.
        helper (str, optional): A helper string providing additional information about the function.
        version (str or None, optional): The version of the function or module.
        test (bool, optional): Flag to indicate if the function is for testing purposes.
        restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
        api (bool, optional): Flag to indicate if the function is part of an API.
        initial (bool, optional): Flag to indicate if the function should be executed at initialization.
        exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
        test_only (bool, optional): Flag to indicate if the function should only be used for testing.
        memory_cache (bool, optional): Flag to enable memory caching for the function.
        request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
        file_cache (bool, optional): Flag to enable file caching for the function.
        row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
        state (bool or None, optional): Flag to indicate if the function maintains state.
        level (int, optional): The level of the function, used for prioritization or categorization.
        memory_cache_max_size (int, optional): Maximum size of the memory cache.
        memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
        samples (list or dict or None, optional): Samples or examples of function usage.
        interface (str, optional): The interface type for the function.
        pre_compute (callable, optional): A function to be called before the main function.
        post_compute (callable, optional): A function to be called after the main function.
        api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
        websocket_handler (str, optional): The name of the websocket handler to use.

    Returns:
        function: The decorated function with additional processing and registration capabilities.
    """
        if interface is None:
            interface = "tb"
        if test_only and 'test' not in self.id:
            return lambda *args, **kwargs: args
        return self._create_decorator(interface,
                                      name,
                                      mod_name,
                                      level=level,
                                      restrict_in_virtual_mode=restrict_in_virtual_mode,
                                      helper=helper,
                                      api=api,
                                      version=version,
                                      initial=initial,
                                      exit_f=exit_f,
                                      test=test,
                                      samples=samples,
                                      state=state,
                                      pre_compute=pre_compute,
                                      post_compute=post_compute,
                                      memory_cache=memory_cache,
                                      file_cache=file_cache,
                                      request_as_kwarg=request_as_kwarg,
                                      row=row,
                                      api_methods=api_methods,
                                      memory_cache_max_size=memory_cache_max_size,
                                      memory_cache_ttl=memory_cache_ttl,
                                      websocket_handler=websocket_handler,
                                      )

    def save_autocompletion_dict(self):
        autocompletion_dict = {}
        for module_name, _module in self.functions.items():
            data = {}
            for function_name, function_data in self.functions[module_name].items():
                if not isinstance(function_data, dict):
                    continue
                data[function_name] = {arg: None for arg in
                                       function_data.get("params", [])}
                if len(data[function_name].keys()) == 0:
                    data[function_name] = None
            autocompletion_dict[module_name] = data if len(data.keys()) > 0 else None
        self.config_fh.add_to_save_file_handler("auto~~~~~~", str(autocompletion_dict))

    def get_autocompletion_dict(self):
        return self.config_fh.get_file_handler("auto~~~~~~")

    def save_registry_as_enums(self, directory: str, filename: str):
        # Ordner erstellen, falls nicht vorhanden
        if not os.path.exists(directory):
            os.makedirs(directory)

        # Dateipfad vorbereiten
        filepath = os.path.join(directory, filename)

        # Enum-Klassen als Strings generieren
        enum_classes = [f'"""Automatic generated by ToolBox v = {self.version}"""'
                        f'\nfrom enum import Enum\nfrom dataclasses import dataclass'
                        f'\n\n\n']
        for module, functions in self.functions.items():
            if module.startswith("APP_INSTANCE"):
                continue
            class_name = module
            enum_members = "\n    ".join(
                [
                    f"{func_name.upper().replace('-', '')}"
                    f" = '{func_name}' "
                    f"# Input: ({fuction_data['params'] if isinstance(fuction_data, dict) else ''}),"
                    f" Output: {fuction_data['signature'].return_annotation if isinstance(fuction_data, dict) else 'None'}"
                    for func_name, fuction_data in functions.items()])
            enum_class = (f'@dataclass\nclass {class_name.upper().replace(".", "_").replace("-", "")}(Enum):'
                          f"\n    NAME = '{class_name}'\n    {enum_members}")
            enum_classes.append(enum_class)

        # Enums in die Datei schreiben
        data = "\n\n\n".join(enum_classes)
        if len(data) < 12:
            raise ValueError(
                "Invalid Enums Loosing content pleas delete it ur self in the (utils/system/all_functions_enums.py) or add mor new stuff :}")
        with open(filepath, 'w') as file:
            file.write(data)

        print(Style.Bold(Style.BLUE(f"Enums gespeichert in {filepath}")))


    # WS logic

    def _set_rust_ws_bridge(self, bridge_object: Any):
        """
        Diese Methode wird von Rust aufgerufen, um die Kommunikationsbrücke zu setzen.
        Sie darf NICHT manuell von Python aus aufgerufen werden.
        """
        self.print(f"Rust WebSocket bridge has been set for instance {self.id}.")
        self._rust_ws_bridge = bridge_object

    async def ws_send(self, conn_id: str, payload: dict):
        """
        Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

        Args:
            conn_id: Die eindeutige ID der Zielverbindung.
            payload: Ein Dictionary, das als JSON gesendet wird.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
            await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
        except Exception as e:
            self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

    async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
        """
        Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

        Args:
            channel_id: Der Kanal, an den gesendet werden soll.
            payload: Ein Dictionary, das als JSON gesendet wird.
            source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
        """
        if self._rust_ws_bridge is None:
            self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
            return

        try:
            # Ruft die asynchrone Rust-Broadcast-Methode auf
            await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
        except Exception as e:
            self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)
disconnect(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
244
245
246
@staticmethod
def disconnect(*args, **kwargs):
    """proxi attr"""
exit_main(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
232
233
234
@staticmethod
def exit_main(*args, **kwargs):
    """proxi attr"""
get_function(name, **kwargs)

Kwargs for _get_function metadata:: return the registered function dictionary stateless: (function_data, None), 0 stateful: (function_data, higher_order_function), 0 state::boolean specification::str default app

Source code in toolboxv2/utils/toolbox.py
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
def get_function(self, name: Enum or tuple, **kwargs):
    """
    Kwargs for _get_function
        metadata:: return the registered function dictionary
            stateless: (function_data, None), 0
            stateful: (function_data, higher_order_function), 0
        state::boolean
            specification::str default app
    """
    if isinstance(name, tuple):
        return self._get_function(None, as_str=name, **kwargs)
    else:
        return self._get_function(name, **kwargs)
hide_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
236
237
238
@staticmethod
def hide_console(*args, **kwargs):
    """proxi attr"""
init_mod(mod_name, spec='app')

Initializes a module in a thread-safe manner by submitting the asynchronous initialization to the running event loop.

Source code in toolboxv2/utils/toolbox.py
625
626
627
628
629
630
631
632
def init_mod(self, mod_name, spec='app'):
    """
    Initializes a module in a thread-safe manner by submitting the
    asynchronous initialization to the running event loop.
    """
    if '.' in mod_name:
        mod_name = mod_name.split('.')[0]
    self.run_bg_task(self.a_init_mod, mod_name, spec)
run(*args, request=None, running_function_coro=None, **kwargs)

Run a function with support for SSE streaming in both threaded and non-threaded contexts.

Source code in toolboxv2/utils/toolbox.py
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
def run(self, *args, request=None, running_function_coro=None, **kwargs):
    """
    Run a function with support for SSE streaming in both
    threaded and non-threaded contexts.
    """
    if running_function_coro is None:
        mn, fn = args[0]
        if self.functions.get(mn, {}).get(fn, {}).get('request_as_kwarg', False):
            kwargs["request"] = RequestData.from_dict(request)
            if 'data' in kwargs and 'data' not in self.functions.get(mn, {}).get(fn, {}).get('params', []):
                kwargs["request"].data = kwargs["request"].body = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in self.functions.get(mn, {}).get(fn, {}).get('params',
                                                                                                       []):
                kwargs["request"].form_data = kwargs["request"].body = kwargs['form_data']
                del kwargs['form_data']
        else:
            params = self.functions.get(mn, {}).get(fn, {}).get('params', [])
            # auto pars data and form_data to kwargs by key
            do = False
            data = {}
            if 'data' in kwargs and 'data' not in params:
                do = True
                data = kwargs['data']
                del kwargs['data']
            if 'form_data' in kwargs and 'form_data' not in params:
                do = True
                data = kwargs['form_data']
                del kwargs['form_data']
            if do:
                for k in params:
                    if k in data:
                        kwargs[k] = data[k]
                        del data[k]

    # Create the coroutine
    coro = running_function_coro or self.a_run_any(*args, **kwargs)

    # Get or create an event loop
    try:
        loop = asyncio.get_event_loop()
        is_running = loop.is_running()
    except RuntimeError:
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        is_running = False

    # If the loop is already running, run in a separate thread
    if is_running:
        # Create thread pool executor as needed
        if not hasattr(self.__class__, '_executor'):
            self.__class__._executor = ThreadPoolExecutor(max_workers=4)

        def run_in_new_thread():
            # Set up a new loop in this thread
            new_loop = asyncio.new_event_loop()
            asyncio.set_event_loop(new_loop)

            try:
                # Run the coroutine
                return new_loop.run_until_complete(coro)
            finally:
                new_loop.close()

        # Run in thread and get result
        thread_result = self.__class__._executor.submit(run_in_new_thread).result()

        # Handle streaming results from thread
        if isinstance(thread_result, dict) and thread_result.get("is_stream"):
            # Create a new SSE stream in the main thread
            async def stream_from_function():
                # Re-run the function with direct async access
                stream_result = await self.a_run_any(*args, **kwargs)

                if (isinstance(stream_result, Result) and
                    getattr(stream_result.result, 'data_type', None) == "stream"):
                    # Get and forward data from the original generator
                    original_gen = stream_result.result.data.get("generator")
                    if inspect.isasyncgen(original_gen):
                        async for item in original_gen:
                            yield item

            # Return a new streaming Result
            return Result.stream(
                stream_generator=stream_from_function(),
                headers=thread_result.get("headers", {})
            )

        result = thread_result
    else:
        # Direct execution when loop is not running
        result = loop.run_until_complete(coro)

    # Process the final result
    if isinstance(result, Result):
        if 'debug' in self.id:
            result.print()
        if getattr(result.result, 'data_type', None) == "stream":
            return result
        return result.to_api_result().model_dump(mode='json')

    return result
run_bg_task(task, *args, **kwargs)

Runs a coroutine in the background without blocking the caller.

This is the primary method for "fire-and-forget" async tasks. It schedules the coroutine to run on the application's main event loop.

Parameters:

Name Type Description Default
task Callable

The coroutine function to run.

required
*args

Arguments to pass to the coroutine function.

()
**kwargs

Keyword arguments to pass to the coroutine function.

{}

Returns:

Type Description
Task | None

An asyncio.Task object representing the scheduled task, or None if

Task | None

the task could not be scheduled.

Source code in toolboxv2/utils/toolbox.py
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
def run_bg_task(self, task: Callable, *args, **kwargs) -> asyncio.Task | None:
    """
    Runs a coroutine in the background without blocking the caller.

    This is the primary method for "fire-and-forget" async tasks. It schedules
    the coroutine to run on the application's main event loop.

    Args:
        task: The coroutine function to run.
        *args: Arguments to pass to the coroutine function.
        **kwargs: Keyword arguments to pass to the coroutine function.

    Returns:
        An asyncio.Task object representing the scheduled task, or None if
        the task could not be scheduled.
    """
    if not callable(task):
        self.logger.warning("Task passed to run_bg_task is not callable!")
        return None

    if not asyncio.iscoroutinefunction(task) and not asyncio.iscoroutine(task):
        self.logger.warning(f"Task '{getattr(task, '__name__', 'unknown')}' is not a coroutine. "
                            f"Use run_bg_task_advanced for synchronous functions.")
        # Fallback to advanced runner for convenience
        self.run_bg_task_advanced(task, *args, **kwargs)
        return None

    try:
        loop = self.loop_gard()
        if not loop.is_running():
            # If the main loop isn't running, we can't create a task on it.
            # This scenario is handled by run_bg_task_advanced.
            self.logger.info("Main event loop not running. Delegating to advanced background runner.")
            return self.run_bg_task_advanced(task, *args, **kwargs)

        # Create the coroutine if it's a function
        coro = task(*args, **kwargs) if asyncio.iscoroutinefunction(task) else task

        # Create a task on the running event loop
        bg_task = loop.create_task(coro)

        # Add a callback to log exceptions from the background task
        def _log_exception(the_task: asyncio.Task):
            if not the_task.cancelled() and the_task.exception():
                self.logger.error(f"Exception in background task '{the_task.get_name()}':",
                                  exc_info=the_task.exception())

        bg_task.add_done_callback(_log_exception)
        self.bg_tasks.append(bg_task)
        return bg_task

    except Exception as e:
        self.logger.error(f"Failed to schedule background task: {e}", exc_info=True)
        return None
run_bg_task_advanced(task, *args, **kwargs)

Runs a task in a separate, dedicated background thread with its own event loop.

This is ideal for: 1. Running an async task from a synchronous context. 2. Launching a long-running, independent operation that should not interfere with the main application's event loop.

Parameters:

Name Type Description Default
task Callable

The function to run (can be sync or async).

required
*args

Arguments for the task.

()
**kwargs

Keyword arguments for the task.

{}

Returns:

Type Description
Thread

The threading.Thread object managing the background execution.

Source code in toolboxv2/utils/toolbox.py
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
def run_bg_task_advanced(self, task: Callable, *args, **kwargs) -> threading.Thread:
    """
    Runs a task in a separate, dedicated background thread with its own event loop.

    This is ideal for:
    1. Running an async task from a synchronous context.
    2. Launching a long-running, independent operation that should not
       interfere with the main application's event loop.

    Args:
        task: The function to run (can be sync or async).
        *args: Arguments for the task.
        **kwargs: Keyword arguments for the task.

    Returns:
        The threading.Thread object managing the background execution.
    """
    if not callable(task):
        self.logger.warning("Task for run_bg_task_advanced is not callable!")
        return None

    def thread_target():
        # Each thread gets its own event loop.
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

        try:
            # Prepare the coroutine we need to run
            if asyncio.iscoroutinefunction(task):
                coro = task(*args, **kwargs)
            elif asyncio.iscoroutine(task):
                # It's already a coroutine object
                coro = task
            else:
                # It's a synchronous function, run it in an executor
                # to avoid blocking the new event loop.
                coro = loop.run_in_executor(None, lambda: task(*args, **kwargs))

            # Run the coroutine to completion
            result = loop.run_until_complete(coro)
            self.logger.debug(f"Advanced background task '{getattr(task, '__name__', 'unknown')}' completed.")
            if result is not None:
                self.logger.debug(f"Task result: {str(result)[:100]}")

        except Exception as e:
            self.logger.error(f"Error in advanced background task '{getattr(task, '__name__', 'unknown')}':",
                              exc_info=e)
        finally:
            # Cleanly shut down the event loop in this thread.
            try:
                all_tasks = asyncio.all_tasks(loop=loop)
                if all_tasks:
                    for t in all_tasks:
                        t.cancel()
                    loop.run_until_complete(asyncio.gather(*all_tasks, return_exceptions=True))
            finally:
                loop.close()
                asyncio.set_event_loop(None)

    # Create, start, and return the thread.
    # It's a daemon thread so it won't prevent the main app from exiting.
    t = threading.Thread(target=thread_target, daemon=True, name=f"BGTask-{getattr(task, '__name__', 'unknown')}")
    self.bg_tasks.append(t)
    t.start()
    return t
show_console(*args, **kwargs) staticmethod

proxi attr

Source code in toolboxv2/utils/toolbox.py
240
241
242
@staticmethod
def show_console(*args, **kwargs):
    """proxi attr"""
tb(name=None, mod_name='', helper='', version=None, test=True, restrict_in_virtual_mode=False, api=False, initial=False, exit_f=False, test_only=False, memory_cache=False, file_cache=False, request_as_kwarg=False, row=False, state=None, level=-1, memory_cache_max_size=100, memory_cache_ttl=300, samples=None, interface=None, pre_compute=None, post_compute=None, api_methods=None, websocket_handler=None)

A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Parameters:

Name Type Description Default
name str

The name to register the function under. Defaults to the function's own name.

None
mod_name str

The name of the module the function belongs to.

''
helper str

A helper string providing additional information about the function.

''
version str or None

The version of the function or module.

None
test bool

Flag to indicate if the function is for testing purposes.

True
restrict_in_virtual_mode bool

Flag to restrict the function in virtual mode.

False
api bool

Flag to indicate if the function is part of an API.

False
initial bool

Flag to indicate if the function should be executed at initialization.

False
exit_f bool

Flag to indicate if the function should be executed at exit.

False
test_only bool

Flag to indicate if the function should only be used for testing.

False
memory_cache bool

Flag to enable memory caching for the function.

False
request_as_kwarg bool

Flag to get request if the fuction is calld from api.

False
file_cache bool

Flag to enable file caching for the function.

False
row bool

rather to auto wrap the result in Result type default False means no row data aka result type

False
state bool or None

Flag to indicate if the function maintains state.

None
level int

The level of the function, used for prioritization or categorization.

-1
memory_cache_max_size int

Maximum size of the memory cache.

100
memory_cache_ttl int

Time-to-live for the memory cache entries.

300
samples list or dict or None

Samples or examples of function usage.

None
interface str

The interface type for the function.

None
pre_compute callable

A function to be called before the main function.

None
post_compute callable

A function to be called after the main function.

None
api_methods list[str]

default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.

None
websocket_handler str

The name of the websocket handler to use.

None

Returns:

Name Type Description
function

The decorated function with additional processing and registration capabilities.

Source code in toolboxv2/utils/toolbox.py
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
def tb(self, name=None,
       mod_name: str = "",
       helper: str = "",
       version: str | None = None,
       test: bool = True,
       restrict_in_virtual_mode: bool = False,
       api: bool = False,
       initial: bool = False,
       exit_f: bool = False,
       test_only: bool = False,
       memory_cache: bool = False,
       file_cache: bool = False,
       request_as_kwarg: bool = False,
       row: bool = False,
       state: bool | None = None,
       level: int = -1,
       memory_cache_max_size: int = 100,
       memory_cache_ttl: int = 300,
       samples: list or dict or None = None,
       interface: ToolBoxInterfaces or None or str = None,
       pre_compute=None,
       post_compute=None,
       api_methods=None,
       websocket_handler: str | None = None,
       ):
    """
A decorator for registering and configuring functions within a module.

This decorator is used to wrap functions with additional functionality such as caching, API conversion, and lifecycle management (initialization and exit). It also handles the registration of the function in the module's function registry.

Args:
    name (str, optional): The name to register the function under. Defaults to the function's own name.
    mod_name (str, optional): The name of the module the function belongs to.
    helper (str, optional): A helper string providing additional information about the function.
    version (str or None, optional): The version of the function or module.
    test (bool, optional): Flag to indicate if the function is for testing purposes.
    restrict_in_virtual_mode (bool, optional): Flag to restrict the function in virtual mode.
    api (bool, optional): Flag to indicate if the function is part of an API.
    initial (bool, optional): Flag to indicate if the function should be executed at initialization.
    exit_f (bool, optional): Flag to indicate if the function should be executed at exit.
    test_only (bool, optional): Flag to indicate if the function should only be used for testing.
    memory_cache (bool, optional): Flag to enable memory caching for the function.
    request_as_kwarg (bool, optional): Flag to get request if the fuction is calld from api.
    file_cache (bool, optional): Flag to enable file caching for the function.
    row (bool, optional): rather to auto wrap the result in Result type default False means no row data aka result type
    state (bool or None, optional): Flag to indicate if the function maintains state.
    level (int, optional): The level of the function, used for prioritization or categorization.
    memory_cache_max_size (int, optional): Maximum size of the memory cache.
    memory_cache_ttl (int, optional): Time-to-live for the memory cache entries.
    samples (list or dict or None, optional): Samples or examples of function usage.
    interface (str, optional): The interface type for the function.
    pre_compute (callable, optional): A function to be called before the main function.
    post_compute (callable, optional): A function to be called after the main function.
    api_methods (list[str], optional): default ["AUTO"] (GET if not params, POST if params) , GET, POST, PUT or DELETE.
    websocket_handler (str, optional): The name of the websocket handler to use.

Returns:
    function: The decorated function with additional processing and registration capabilities.
"""
    if interface is None:
        interface = "tb"
    if test_only and 'test' not in self.id:
        return lambda *args, **kwargs: args
    return self._create_decorator(interface,
                                  name,
                                  mod_name,
                                  level=level,
                                  restrict_in_virtual_mode=restrict_in_virtual_mode,
                                  helper=helper,
                                  api=api,
                                  version=version,
                                  initial=initial,
                                  exit_f=exit_f,
                                  test=test,
                                  samples=samples,
                                  state=state,
                                  pre_compute=pre_compute,
                                  post_compute=post_compute,
                                  memory_cache=memory_cache,
                                  file_cache=file_cache,
                                  request_as_kwarg=request_as_kwarg,
                                  row=row,
                                  api_methods=api_methods,
                                  memory_cache_max_size=memory_cache_max_size,
                                  memory_cache_ttl=memory_cache_ttl,
                                  websocket_handler=websocket_handler,
                                  )
wait_for_bg_tasks(timeout=None)

Wait for all background tasks to complete.

Parameters:

Name Type Description Default
timeout

Maximum time to wait (in seconds) for all tasks to complete. None means wait indefinitely.

None

Returns:

Name Type Description
bool

True if all tasks completed, False if timeout occurred

Source code in toolboxv2/utils/toolbox.py
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
def wait_for_bg_tasks(self, timeout=None):
    """
    Wait for all background tasks to complete.

    Args:
        timeout: Maximum time to wait (in seconds) for all tasks to complete.
                 None means wait indefinitely.

    Returns:
        bool: True if all tasks completed, False if timeout occurred
    """
    active_tasks = [t for t in self.bg_tasks if t.is_alive()]

    for task in active_tasks:
        task.join(timeout=timeout)
        if task.is_alive():
            return False

    return True
ws_broadcast(channel_id, payload, source_conn_id='python_broadcast') async

Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

Parameters:

Name Type Description Default
channel_id str

Der Kanal, an den gesendet werden soll.

required
payload dict

Ein Dictionary, das als JSON gesendet wird.

required
source_conn_id optional

Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.

'python_broadcast'
Source code in toolboxv2/utils/toolbox.py
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
async def ws_broadcast(self, channel_id: str, payload: dict, source_conn_id: str = "python_broadcast"):
    """
    Sendet eine Nachricht asynchron an alle Clients in einem Kanal/Raum.

    Args:
        channel_id: Der Kanal, an den gesendet werden soll.
        payload: Ein Dictionary, das als JSON gesendet wird.
        source_conn_id (optional): Die ID der ursprünglichen Verbindung, um Echos zu vermeiden.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot broadcast WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Broadcast-Methode auf
        await self._rust_ws_bridge.broadcast_message(channel_id, json.dumps(payload), source_conn_id)
    except Exception as e:
        self.logger.error(f"Failed to broadcast WebSocket message to channel {channel_id}: {e}", exc_info=True)
ws_send(conn_id, payload) async

Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

Parameters:

Name Type Description Default
conn_id str

Die eindeutige ID der Zielverbindung.

required
payload dict

Ein Dictionary, das als JSON gesendet wird.

required
Source code in toolboxv2/utils/toolbox.py
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
async def ws_send(self, conn_id: str, payload: dict):
    """
    Sendet eine Nachricht asynchron an eine einzelne WebSocket-Verbindung.

    Args:
        conn_id: Die eindeutige ID der Zielverbindung.
        payload: Ein Dictionary, das als JSON gesendet wird.
    """
    if self._rust_ws_bridge is None:
        self.logger.error("Cannot send WebSocket message: Rust bridge is not initialized.")
        return

    try:
        # Ruft die asynchrone Rust-Methode auf und wartet auf deren Abschluss
        await self._rust_ws_bridge.send_message(conn_id, json.dumps(payload))
    except Exception as e:
        self.logger.error(f"Failed to send WebSocket message to {conn_id}: {e}", exc_info=True)

toolboxv2.show_console(show=True)

Source code in toolboxv2/utils/extras/show_and_hide_console.py
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
def show_console(show=True):
    global TBRUNNER_console_viabel
    """Brings up the Console Window."""
    try:
        if show and not TBRUNNER_console_viabel:
            # Show console
            ctypes.windll.user32.ShowWindow(ctypes.windll.kernel32.GetConsoleWindow(), 4)
            TBRUNNER_console_viabel = True
            return True
        elif not show and TBRUNNER_console_viabel:
            # Hide console
            ctypes.windll.user32.ShowWindow(ctypes.windll.kernel32.GetConsoleWindow(), 0)
            TBRUNNER_console_viabel = False
            return True
    except:
        print(f"Could not show_console {show=}", )
        return False
    return False

Logging

toolboxv2.get_logger()

Source code in toolboxv2/utils/system/tb_logger.py
136
137
def get_logger() -> logging.Logger:
    return logging.getLogger(loggerNameOfToolboxv2)

toolboxv2.setup_logging(level, name=loggerNameOfToolboxv2, online_level=None, is_online=False, file_level=None, interminal=False, logs_directory='../logs', app_name='main')

Source code in toolboxv2/utils/system/tb_logger.py
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
def setup_logging(level: int, name=loggerNameOfToolboxv2, online_level=None, is_online=False, file_level=None,
                  interminal=False, logs_directory="../logs", app_name="main"):
    global loggerNameOfToolboxv2

    if not online_level:
        online_level = level

    if not file_level:
        file_level = level

    if not os.path.exists(logs_directory):
        os.makedirs(logs_directory, exist_ok=True)
    if not os.path.exists(logs_directory + "/Logs.info"):
        open(f"{logs_directory}/Logs.info", "a").close()

    loggerNameOfToolboxv2 = name

    available_log_levels = [logging.CRITICAL, logging.FATAL, logging.ERROR, logging.WARNING, logging.WARN, logging.INFO,
                            logging.DEBUG, logging.NOTSET]

    if level not in available_log_levels:
        raise ValueError(f"level must be one of {available_log_levels}, but logging level is {level}")

    if online_level not in available_log_levels:
        raise ValueError(f"online_level must be one of {available_log_levels}, but logging level is {online_level}")

    if file_level not in available_log_levels:
        raise ValueError(f"file_level must be one of {available_log_levels}, but logging level is {file_level}")

    log_date = datetime.datetime.today().strftime('%Y-%m-%d')
    log_levels = ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"]
    log_level_index = log_levels.index(logging.getLevelName(level))

    filename = f"Logs-{name}-{log_date}-{log_levels[log_level_index]}"
    log_filename = f"{logs_directory}/{filename}.log"

    log_info_data = {
        filename: 0,
        "H": "localhost",
        "P": 62435
    }

    with open(f"{logs_directory}/Logs.info") as li:
        log_info_data_str = li.read()
        try:
            log_info_data = eval(log_info_data_str)
        except SyntaxError:
            if log_info_data_str:
                print(Style.RED(Style.Bold("Could not parse log info data")))

        if filename not in log_info_data:
            log_info_data[filename] = 0

        if not os.path.exists(log_filename):
            log_info_data[filename] = 0
            print("new log file")

        if os.path.exists(log_filename):
            log_info_data[filename] += 1

            while os.path.exists(f"{logs_directory}/{filename}#{log_info_data[filename]}.log"):
                log_info_data[filename] += 1

            try:
                os.rename(log_filename,
                          f"{logs_directory}/{filename}#{log_info_data[filename]}.log")
            except PermissionError:
                print(Style.YELLOW(Style.Bold(f"Could not rename log file appending on {filename}")))

    with open(f"{logs_directory}/Logs.info", "w") as li:
        if len(log_info_data.keys()) >= 7:
            log_info_data = {
                filename: log_info_data[filename],
                "H": log_info_data["H"],
                "P": log_info_data["P"]
            }
        li.write(str(log_info_data))

    try:
        with open(log_filename, "a"):
            pass
    except OSError:
        log_filename = f"{logs_directory}/Logs-Test-{log_date}-{log_levels[log_level_index]}.log"
        with open(log_filename, "a"):
            pass

    logger = logging.getLogger(name)

    logger.setLevel(level)
    # Prevent logger from propagating to parent loggers
    logger.propagate = False

    terminal_format = f"{app_name} %(asctime)s %(levelname)s %(name)s - %(message)s"
    file_format = f"{app_name} %(asctime)s - %(name)s - %(levelname)s - %(filename)s - %(funcName)s:%(lineno)d - %(message)s"

    # Configure handlers
    handlers = []

    # File handler (always added)
    file_handler = logging.FileHandler(log_filename)
    file_handler.setFormatter(logging.Formatter(file_format))
    file_handler.setLevel(file_level)
    handlers.append(file_handler)

    # Terminal handler (if requested)
    if interminal:
        terminal_handler = logging.StreamHandler()
        terminal_handler.setFormatter(logging.Formatter(terminal_format))
        terminal_handler.setLevel(level)
        handlers.append(terminal_handler)

    # Socket handler (if requested)
    if is_online:
        socket_handler = SocketHandler(log_info_data["H"], log_info_data["P"])
        socket_handler.setFormatter(logging.Formatter(file_format))
        socket_handler.setLevel(online_level)
        handlers.append(socket_handler)

    # Add all handlers to logger
    for handler in handlers:
        logger.addHandler(handler)

    return logger, filename

Styling & Console Output

toolboxv2.Style

Source code in toolboxv2/utils/extras/Style.py
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
class Style:
    _END = '\33[0m'
    _BLACK = '\33[30m'
    _RED = '\33[31m'
    _GREEN = '\33[32m'
    _YELLOW = '\33[33m'
    _BLUE = '\33[34m'
    _MAGENTA = '\33[35m'
    _CYAN = '\33[36m'
    _WHITE = '\33[37m'

    _Bold = '\33[1m'
    _ITALIC = '\33[3m'
    _Underline = '\33[4m'
    _BLINK = '\33[5m'
    _BLINK2 = '\33[6m'
    _Reversed = '\33[7m'

    _BLACKBG = '\33[40m'
    _REDBG = '\33[41m'
    _GREENBG = '\33[42m'
    _YELLOWBG = '\33[43m'
    _BLUEBG = '\33[44m'
    _VIOLETBG = '\33[45m'
    _BEIGEBG = '\33[46m'
    _WHITEBG = '\33[47m'

    _GREY = '\33[90m'
    _RED2 = '\33[91m'
    _GREEN2 = '\33[92m'
    _YELLOW2 = '\33[93m'
    _BLUE2 = '\33[94m'
    _VIOLET2 = '\33[95m'
    _BEIGE2 = '\33[96m'
    _WHITE2 = '\33[97m'

    _GREYBG = '\33[100m'
    _REDBG2 = '\33[101m'
    _GREENBG2 = '\33[102m'
    _YELLOWBG2 = '\33[103m'
    _BLUEBG2 = '\33[104m'
    _VIOLETBG2 = '\33[105m'
    _BEIGEBG2 = '\33[106m'
    _WHITEBG2 = '\33[107m'

    style_dic = {
        "END": _END,
        "BLACK": _BLACK,
        "RED": _RED,
        "GREEN": _GREEN,
        "YELLOW": _YELLOW,
        "BLUE": _BLUE,
        "MAGENTA": _MAGENTA,
        "CYAN": _CYAN,
        "WHITE": _WHITE,
        "Bold": _Bold,
        "Underline": _Underline,
        "Reversed": _Reversed,

        "ITALIC": _ITALIC,
        "BLINK": _BLINK,
        "BLINK2": _BLINK2,
        "BLACKBG": _BLACKBG,
        "REDBG": _REDBG,
        "GREENBG": _GREENBG,
        "YELLOWBG": _YELLOWBG,
        "BLUEBG": _BLUEBG,
        "VIOLETBG": _VIOLETBG,
        "BEIGEBG": _BEIGEBG,
        "WHITEBG": _WHITEBG,
        "GREY": _GREY,
        "RED2": _RED2,
        "GREEN2": _GREEN2,
        "YELLOW2": _YELLOW2,
        "BLUE2": _BLUE2,
        "VIOLET2": _VIOLET2,
        "BEIGE2": _BEIGE2,
        "WHITE2": _WHITE2,
        "GREYBG": _GREYBG,
        "REDBG2": _REDBG2,
        "GREENBG2": _GREENBG2,
        "YELLOWBG2": _YELLOWBG2,
        "BLUEBG2": _BLUEBG2,
        "VIOLETBG2": _VIOLETBG2,
        "BEIGEBG2": _BEIGEBG2,
        "WHITEBG2": _WHITEBG2,

    }

    @staticmethod
    @text_save
    def END_():
        print(Style._END)

    @staticmethod
    @text_save
    def GREEN_():
        print(Style._GREEN)

    @staticmethod
    @text_save
    def BLUE(text: str):
        return Style._BLUE + text + Style._END

    @staticmethod
    @text_save
    def BLACK(text: str):
        return Style._BLACK + text + Style._END

    @staticmethod
    @text_save
    def RED(text: str):
        return Style._RED + text + Style._END

    @staticmethod
    @text_save
    def GREEN(text: str):
        return Style._GREEN + text + Style._END

    @staticmethod
    @text_save
    def YELLOW(text: str):
        return Style._YELLOW + text + Style._END

    @staticmethod
    @text_save
    def MAGENTA(text: str):
        return Style._MAGENTA + text + Style._END

    @staticmethod
    @text_save
    def CYAN(text: str):
        return Style._CYAN + text + Style._END

    @staticmethod
    @text_save
    def WHITE(text: str):
        return Style._WHITE + text + Style._END

    @staticmethod
    @text_save
    def Bold(text: str):
        return Style._Bold + text + Style._END

    @staticmethod
    @text_save
    def Underline(text: str):
        return Style._Underline + text + Style._END

    @staticmethod
    @text_save
    def Underlined(text: str):
        return Style._Underline + text + Style._END

    @staticmethod
    @text_save
    def Reversed(text: str):
        return Style._Reversed + text + Style._END

    @staticmethod
    @text_save
    def ITALIC(text: str):
        return Style._ITALIC + text + Style._END

    @staticmethod
    @text_save
    def BLINK(text: str):
        return Style._BLINK + text + Style._END

    @staticmethod
    @text_save
    def BLINK2(text: str):
        return Style._BLINK2 + text + Style._END

    @staticmethod
    @text_save
    def BLACKBG(text: str):
        return Style._BLACKBG + text + Style._END

    @staticmethod
    @text_save
    def REDBG(text: str):
        return Style._REDBG + text + Style._END

    @staticmethod
    @text_save
    def GREENBG(text: str):
        return Style._GREENBG + text + Style._END

    @staticmethod
    @text_save
    def YELLOWBG(text: str):
        return Style._YELLOWBG + text + Style._END

    @staticmethod
    @text_save
    def BLUEBG(text: str):
        return Style._BLUEBG + text + Style._END

    @staticmethod
    @text_save
    def VIOLETBG(text: str):
        return Style._VIOLETBG + text + Style._END

    @staticmethod
    @text_save
    def BEIGEBG(text: str):
        return Style._BEIGEBG + text + Style._END

    @staticmethod
    @text_save
    def WHITEBG(text: str):
        return Style._WHITEBG + text + Style._END

    @staticmethod
    @text_save
    def GREY(text: str):
        return Style._GREY + str(text) + Style._END

    @staticmethod
    @text_save
    def RED2(text: str):
        return Style._RED2 + text + Style._END

    @staticmethod
    @text_save
    def GREEN2(text: str):
        return Style._GREEN2 + text + Style._END

    @staticmethod
    @text_save
    def YELLOW2(text: str):
        return Style._YELLOW2 + text + Style._END

    @staticmethod
    @text_save
    def BLUE2(text: str):
        return Style._BLUE2 + text + Style._END

    @staticmethod
    @text_save
    def VIOLET2(text: str):
        return Style._VIOLET2 + text + Style._END

    @staticmethod
    @text_save
    def BEIGE2(text: str):
        return Style._BEIGE2 + text + Style._END

    @staticmethod
    @text_save
    def WHITE2(text: str):
        return Style._WHITE2 + text + Style._END

    @staticmethod
    @text_save
    def GREYBG(text: str):
        return Style._GREYBG + text + Style._END

    @staticmethod
    @text_save
    def REDBG2(text: str):
        return Style._REDBG2 + text + Style._END

    @staticmethod
    @text_save
    def GREENBG2(text: str):
        return Style._GREENBG2 + text + Style._END

    @staticmethod
    @text_save
    def YELLOWBG2(text: str):
        return Style._YELLOWBG2 + text + Style._END

    @staticmethod
    @text_save
    def BLUEBG2(text: str):
        return Style._BLUEBG2 + text + Style._END

    @staticmethod
    @text_save
    def VIOLETBG2(text: str):
        return Style._VIOLETBG2 + text + Style._END

    @staticmethod
    @text_save
    def BEIGEBG2(text: str):
        return Style._BEIGEBG2 + text + Style._END

    @staticmethod
    @text_save
    def WHITEBG2(text: str):
        return Style._WHITEBG2 + text + Style._END

    @staticmethod
    @text_save
    def loading_al(text: str):
        b = f"{text} /"
        print(b)
        sleep(0.05)
        cls()
        b = f"{text} -"
        print(b)
        sleep(0.05)
        cls()
        b = f"{text} \\"
        print(b)
        sleep(0.05)
        cls()
        b = f"{text} |"
        print(b)
        sleep(0.05)
        cls()

    @property
    def END(self):
        return self._END

    def color_demo(self):
        for color in self.style_dic:
            print(f"{color} -> {self.style_dic[color]}Effect{self._END}")

    @property
    def Underline2(self):
        return self._Underline

toolboxv2.Spinner

Enhanced Spinner with tqdm-like line rendering.

Source code in toolboxv2/utils/extras/Style.py
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
class Spinner:
    """
    Enhanced Spinner with tqdm-like line rendering.
    """
    SYMBOL_SETS = {
        "c": ["◐", "◓", "◑", "◒"],
        "b": ["▁", "▃", "▄", "▅", "▆", "▇", "█", "▇", "▆", "▅", "▄", "▃"],
        "d": ["⣾", "⣽", "⣻", "⢿", "⡿", "⣟", "⣯", "⣷"],
        "w": ["🌍", "🌎", "🌏"],
        "s": ["🌀   ", " 🌀  ", "  🌀 ", "   🌀", "  🌀 ", " 🌀  "],
        "+": ["+", "x"],
        "t": ["✶", "✸", "✹", "✺", "✹", "✷"]
    }

    def __init__(
        self,
        message: str = "Loading...",
        delay: float = 0.1,
        symbols=None,
        count_down: bool = False,
        time_in_s: float = 0
    ):
        """Initialize spinner with flexible configuration."""
        # Resolve symbol set.
        if isinstance(symbols, str):
            symbols = self.SYMBOL_SETS.get(symbols, None)

        # Default symbols if not provided.
        if symbols is None:
            symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

        # Test mode symbol set.
        if 'unittest' in sys.argv[0]:
            symbols = ['#', '=', '-']

        self.spinner = itertools.cycle(symbols)
        self.delay = delay
        self.message = message
        self.running = False
        self.spinner_thread = None
        self.max_t = time_in_s
        self.contd = count_down

        # Rendering management.
        self._is_primary = False
        self._start_time = 0

        # Central manager.
        self.manager = SpinnerManager()

    def _generate_render_line(self):
        """Generate the primary render line."""
        current_time = time.time()
        if self.contd:
            remaining = max(0, self.max_t - (current_time - self._start_time))
            time_display = f"{remaining:.2f}"
        else:
            time_display = f"{current_time - self._start_time:.2f}"

        symbol = next(self.spinner)
        return f"{symbol} {self.message} | {time_display}"

    def _generate_secondary_info(self):
        """Generate secondary spinner info for additional spinners."""
        return f"{self.message}"

    def __enter__(self):
        """Start the spinner."""
        self.running = True
        self._start_time = time.time()
        self.manager.register_spinner(self)
        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        """Stop the spinner."""
        self.running = False
        self.manager.unregister_spinner(self)
        # Clear the spinner's line if it was the primary spinner.
        if self._is_primary:
            sys.stdout.write("\r\033[K")
            sys.stdout.flush()

__enter__()

Start the spinner.

Source code in toolboxv2/utils/extras/Style.py
644
645
646
647
648
649
def __enter__(self):
    """Start the spinner."""
    self.running = True
    self._start_time = time.time()
    self.manager.register_spinner(self)
    return self

__exit__(exc_type, exc_value, exc_traceback)

Stop the spinner.

Source code in toolboxv2/utils/extras/Style.py
651
652
653
654
655
656
657
658
def __exit__(self, exc_type, exc_value, exc_traceback):
    """Stop the spinner."""
    self.running = False
    self.manager.unregister_spinner(self)
    # Clear the spinner's line if it was the primary spinner.
    if self._is_primary:
        sys.stdout.write("\r\033[K")
        sys.stdout.flush()

__init__(message='Loading...', delay=0.1, symbols=None, count_down=False, time_in_s=0)

Initialize spinner with flexible configuration.

Source code in toolboxv2/utils/extras/Style.py
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
def __init__(
    self,
    message: str = "Loading...",
    delay: float = 0.1,
    symbols=None,
    count_down: bool = False,
    time_in_s: float = 0
):
    """Initialize spinner with flexible configuration."""
    # Resolve symbol set.
    if isinstance(symbols, str):
        symbols = self.SYMBOL_SETS.get(symbols, None)

    # Default symbols if not provided.
    if symbols is None:
        symbols = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]

    # Test mode symbol set.
    if 'unittest' in sys.argv[0]:
        symbols = ['#', '=', '-']

    self.spinner = itertools.cycle(symbols)
    self.delay = delay
    self.message = message
    self.running = False
    self.spinner_thread = None
    self.max_t = time_in_s
    self.contd = count_down

    # Rendering management.
    self._is_primary = False
    self._start_time = 0

    # Central manager.
    self.manager = SpinnerManager()

toolboxv2.remove_styles(text, infos=False)

Source code in toolboxv2/utils/extras/Style.py
384
385
386
387
388
389
390
391
392
393
394
395
def remove_styles(text: str, infos=False):
    in_ = []
    for key, style in Style.style_dic.items():
        if style in text:
            text = text.replace(style, '')
            if infos:
                in_.append([key for key, st in Style.style_dic.items() if style == st][0])
    if infos:
        if "END" in in_:
            in_.remove('END')
        return text, in_
    return text

Data Types & Structures

toolboxv2.AppArgs

Source code in toolboxv2/utils/system/types.py
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
class AppArgs:
    init = None
    init_file = 'init.config'
    get_version = False
    mm = False
    sm = False
    lm = False
    modi = 'cli'
    kill = False
    remote = False
    remote_direct_key = None
    background_application = False
    background_application_runner = False
    docker = False
    build = False
    install = None
    remove = None
    update = None
    name = 'main'
    port = 5000
    host = '0.0.0.0'
    load_all_mod_in_files = False
    mods_folder = 'toolboxv2.mods.'
    debug = None
    test = None
    profiler = None
    hot_reload = False
    live_application = True
    sysPrint = False
    kwargs = {}
    session = None

    def default(self):
        return self

    def set(self, name, value):
        setattr(self, name, value)
        return self

toolboxv2.Result

Bases: Generic[T]

Source code in toolboxv2/utils/system/types.py
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
class Result(Generic[T]):
    _task = None
    _generic_type: Optional[Type] = None

    def __init__(self,
                 error: ToolBoxError,
                 result: ToolBoxResult,
                 info: ToolBoxInfo,
                 origin: Any | None = None,
                 generic_type: Optional[Type] = None
                 ):
        self.error: ToolBoxError = error
        self.result: ToolBoxResult = result
        self.info: ToolBoxInfo = info
        self.origin = origin
        self._generic_type = generic_type

    def __class_getitem__(cls, item):
        """Enable Result[Type] syntax"""

        class TypedResult(cls):
            _generic_type = item

            def __init__(self, *args, **kwargs):
                super().__init__(*args, **kwargs)
                self._generic_type = item

        return TypedResult

    def typed_get(self, key=None, default=None) -> T:
        """Get data with type validation"""
        data = self.get(key, default)

        if self._generic_type and data is not None:
            # Validate type matches generic parameter
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    async def typed_aget(self, key=None, default=None) -> T:
        """Async get data with type validation"""
        data = await self.aget(key, default)

        if self._generic_type and data is not None:
            if not self._validate_type(data, self._generic_type):
                from toolboxv2 import get_logger
                get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

        return data

    def _validate_type(self, data, expected_type) -> bool:
        """Validate data matches expected type"""
        try:
            # Handle List[Type] syntax
            origin = get_origin(expected_type)
            if origin is list or origin is List:
                if not isinstance(data, list):
                    return False

                # Check list element types if specified
                args = get_args(expected_type)
                if args and data:
                    element_type = args[0]
                    return all(isinstance(item, element_type) for item in data)
                return True

            # Handle other generic types
            elif origin is not None:
                return isinstance(data, origin)

            # Handle regular types
            else:
                return isinstance(data, expected_type)

        except Exception:
            return True  # Skip validation on error

    @classmethod
    def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
        """Create OK result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    @classmethod
    def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
                   status_code=None) -> 'Result[T]':
        """Create JSON result with type information"""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        instance = cls(error=error, info=info_obj, result=result)
        if hasattr(cls, '_generic_type'):
            instance._generic_type = cls._generic_type

        return instance

    def cast_to(self, target_type: Type[T]) -> 'Result[T]':
        """Cast result to different type"""
        new_result = Result(
            error=self.error,
            result=self.result,
            info=self.info,
            origin=self.origin,
            generic_type=target_type
        )
        new_result._generic_type = target_type
        return new_result

    def get_type_info(self) -> Optional[Type]:
        """Get the generic type information"""
        return self._generic_type

    def is_typed(self) -> bool:
        """Check if result has type information"""
        return self._generic_type is not None

    def as_result(self):
        return self

    def as_dict(self):
        return {
            "error":self.error.value if isinstance(self.error, Enum) else self.error,
        "result" : {
            "data_to":self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
            "data_info":self.result.data_info,
            "data":self.result.data,
            "data_type":self.result.data_type
        } if self.result else None,
        "info" : {
            "exec_code" : self.info.exec_code,  # exec_code umwandel in http resposn codes
        "help_text" : self.info.help_text
        } if self.info else None,
        "origin" : self.origin
        }

    def set_origin(self, origin):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = origin
        return self

    def set_dir_origin(self, name, extras="assets/"):
        if self.origin is not None:
            raise ValueError("You cannot Change the origin of a Result!")
        self.origin = f"mods/{name}/{extras}"
        return self

    def is_error(self):
        if _test_is_result(self.result.data):
            return self.result.data.is_error()
        if self.error == ToolBoxError.none:
            return False
        if self.info.exec_code == 0:
            return False
        return self.info.exec_code != 200

    def is_ok(self):
        return not self.is_error()

    def is_data(self):
        return self.result.data is not None

    def to_api_result(self):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResultBM(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfoBM(
                exec_code=self.info.exec_code,  # exec_code umwandel in http resposn codes
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def task(self, task):
        self._task = task
        return self

    @staticmethod
    def result_from_dict(error: str, result: dict, info: dict, origin: list or None or str):
        # print(f" error={self.error}, result= {self.result}, info= {self.info}, origin= {self.origin}")
        return ApiResult(
            error=error if isinstance(error, Enum) else error,
            result=ToolBoxResultBM(
                data_to=result.get('data_to') if isinstance(result.get('data_to'), Enum) else result.get('data_to'),
                data_info=result.get('data_info', '404'),
                data=result.get('data'),
                data_type=result.get('data_type', '404'),
            ) if result else ToolBoxResultBM(
                data_to=ToolBoxInterfaces.cli.value,
                data_info='',
                data='404',
                data_type='404',
            ),
            info=ToolBoxInfoBM(
                exec_code=info.get('exec_code', 404),
                help_text=info.get('help_text', '404')
            ) if info else ToolBoxInfoBM(
                exec_code=404,
                help_text='404'
            ),
            origin=origin
        ).as_result()

    @classmethod
    def stream(cls,
               stream_generator: Any,  # Renamed from source for clarity
               content_type: str = "text/event-stream",  # Default to SSE
               headers: dict | None = None,
               info: str = "OK",
               interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
               cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
        """
        Create a streaming response Result. Handles SSE and other stream types.

        Args:
            stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
            content_type: Content-Type header (default: text/event-stream for SSE).
            headers: Additional HTTP headers for the response.
            info: Help text for the result.
            interface: Interface to send data to.
            cleanup_func: Optional function for cleanup.

        Returns:
            A Result object configured for streaming.
        """
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        final_generator: AsyncGenerator[str, None]

        if content_type == "text/event-stream":
            # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
            # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
            final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

            # Standard SSE headers for the HTTP response itself
            # These will be stored in the Result object. Rust side decides how to use them.
            standard_sse_headers = {
                "Cache-Control": "no-cache",  # SSE specific
                "Connection": "keep-alive",  # SSE specific
                "X-Accel-Buffering": "no",  # Useful for proxies with SSE
                # Content-Type is implicitly text/event-stream, will be in streaming_data below
            }
            all_response_headers = standard_sse_headers.copy()
            if headers:
                all_response_headers.update(headers)
        else:
            # For non-SSE streams.
            # If stream_generator is sync, wrap it to be async.
            # If already async or single item, it will be handled.
            # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
            # For consistency with how SSEGenerator does it, we can wrap sync ones.
            if inspect.isgenerator(stream_generator) or \
                (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
                final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
            elif inspect.isasyncgen(stream_generator):
                final_generator = stream_generator
            else:  # Single item or string
                async def _single_item_gen():
                    yield stream_generator

                final_generator = _single_item_gen()
            all_response_headers = headers if headers else {}

        # Prepare streaming data to be stored in the Result object
        streaming_data = {
            "type": "stream",  # Indicator for Rust side
            "generator": final_generator,
            "content_type": content_type,  # Let Rust know the intended content type
            "headers": all_response_headers  # Intended HTTP headers for the overall response
        }

        result_payload = ToolBoxResult(
            data_to=interface,
            data=streaming_data,
            data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
            data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
        )

        return cls(error=error, info=info_obj, result=result_payload)

    @classmethod
    def sse(cls,
            stream_generator: Any,
            info: str = "OK",
            interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
            cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
            # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
            ):
        """
        Create an Server-Sent Events (SSE) streaming response Result.

        Args:
            stream_generator: A source yielding individual data items. This can be an
                              async generator, sync generator, iterable, or a single item.
                              Each item will be formatted as an SSE event.
            info: Optional help text for the Result.
            interface: Optional ToolBoxInterface to target.
            cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
            #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

        Returns:
            A Result object configured for SSE streaming.
        """
        # Result.stream will handle calling SSEGenerator.create_sse_stream
        # and setting appropriate default headers for SSE when content_type is "text/event-stream".
        return cls.stream(
            stream_generator=stream_generator,
            content_type="text/event-stream",
            # headers=http_headers, # Pass if we add http_headers param
            info=info,
            interface=interface,
            cleanup_func=cleanup_func
        )

    @classmethod
    def default(cls, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=-1, help_text="")
        result = ToolBoxResult(data_to=interface)
        return cls(error=error, info=info, result=result)

    @classmethod
    def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
        """Create a JSON response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=data,
            data_info="JSON response",
            data_type="json"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
        """Create a text response Result with specific content type."""
        if headers is not None:
            return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=text_data,
            data_info="Text response",
            data_type=content_type
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
               interface=ToolBoxInterfaces.remote):
        """Create a binary data response Result."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=0, help_text=info)

        # Create a dictionary with binary data and metadata
        binary_data = {
            "data": data,
            "content_type": content_type,
            "filename": download_name
        }

        result = ToolBoxResult(
            data_to=interface,
            data=binary_data,
            data_info=f"Binary response: {download_name}" if download_name else "Binary response",
            data_type="binary"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
        """Create a file download response Result.

        Args:
            data: File data as bytes or base64 string
            filename: Name of the file for download
            content_type: MIME type of the file (auto-detected if None)
            info: Response info text
            interface: Target interface

        Returns:
            Result object configured for file download
        """
        import base64
        import mimetypes

        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=200, help_text=info)

        # Auto-detect content type if not provided
        if content_type is None:
            content_type, _ = mimetypes.guess_type(filename)
            if content_type is None:
                content_type = "application/octet-stream"

        # Ensure data is base64 encoded string (as expected by Rust server)
        if isinstance(data, bytes):
            base64_data = base64.b64encode(data).decode('utf-8')
        elif isinstance(data, str):
            # Assume it's already base64 encoded
            base64_data = data
        else:
            raise ValueError("File data must be bytes or base64 string")

        result = ToolBoxResult(
            data_to=interface,
            data=base64_data,  # Rust expects base64 string for "file" type
            data_info=f"File download: {filename}",
            data_type="file"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
        """Create a redirect response."""
        error = ToolBoxError.none
        info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

        result = ToolBoxResult(
            data_to=interface,
            data=url,
            data_info="Redirect response",
            data_type="redirect"
        )

        return cls(error=error, info=info_obj, result=result)

    @classmethod
    def ok(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.native):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def html(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.remote, data_type="html",status=200, headers=None, row=False):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=status, help_text=info)
        from ...utils.system.getting_and_closing_app import get_app

        if not row and not '"<div class="main-content""' in data:
            data = f'<div class="main-content frosted-glass">{data}<div>'
        if not row and not get_app().web_context() in data:
            data = get_app().web_context() + data

        if isinstance(headers, dict):
            result = ToolBoxResult(data_to=interface, data={'html':data,'headers':headers}, data_info=data_info,
                                   data_type="special_html")
        else:
            result = ToolBoxResult(data_to=interface, data=data, data_info=data_info,
                                   data_type=data_type if data_type is not None else type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def future(cls, data=None, data_info="", info="OK", interface=ToolBoxInterfaces.future):
        error = ToolBoxError.none
        info = ToolBoxInfo(exec_code=0, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type="future")
        return cls(error=error, info=info, result=result)

    @classmethod
    def custom_error(cls, data=None, data_info="", info="", exec_code=-1, interface=ToolBoxInterfaces.native):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def error(cls, data=None, data_info="", info="", exec_code=450, interface=ToolBoxInterfaces.remote):
        error = ToolBoxError.custom_error
        info = ToolBoxInfo(exec_code=exec_code, help_text=info)
        result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_user_error(cls, info="", exec_code=-3, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.input_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    @classmethod
    def default_internal_error(cls, info="", exec_code=-2, interface=ToolBoxInterfaces.native, data=None):
        error = ToolBoxError.internal_error
        info = ToolBoxInfo(exec_code, info)
        result = ToolBoxResult(data_to=interface, data=data, data_type=type(data).__name__)
        return cls(error=error, info=info, result=result)

    def print(self, show=True, show_data=True, prifix="", full_data=False):
        data = '\n' + f"{((prifix + f'Data_{self.result.data_type}: ' + str(self.result.data) if self.result.data is not None else 'NO Data') if not isinstance(self.result.data, Result) else self.result.data.print(show=False, show_data=show_data, prifix=prifix + '-')) if show_data else 'Data: private'}"
        origin = '\n' + f"{prifix + 'Origin: ' + str(self.origin) if self.origin is not None else 'NO Origin'}"
        text = (f"Function Exec code: {self.info.exec_code}"
                f"\n{prifix}Info's:"
                f" {self.info.help_text} {'<|> ' + str(self.result.data_info) if self.result.data_info is not None else ''}"
                f"{origin}{((data[:100]+'...') if not full_data else (data)) if not data.endswith('NO Data') else ''}\n")
        if not show:
            return text
        print("\n======== Result ========\n" + text + "------- EndOfD -------")
        return self

    def log(self, show_data=True, prifix=""):
        from toolboxv2 import get_logger
        get_logger().debug(self.print(show=False, show_data=show_data, prifix=prifix).replace("\n", " - "))
        return self

    def __str__(self):
        return self.print(show=False, show_data=True)

    def get(self, key=None, default=None):
        data = self.result.data
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    async def aget(self, key=None, default=None):
        if asyncio.isfuture(self.result.data) or asyncio.iscoroutine(self.result.data) or (
            isinstance(self.result.data_to, Enum) and self.result.data_to.name == ToolBoxInterfaces.future.name):
            data = await self.result.data
        else:
            data = self.get(key=None, default=None)
        if isinstance(data, Result):
            return data.get(key=key, default=default)
        if key is not None and isinstance(data, dict):
            return data.get(key, default)
        return data if data is not None else default

    def lazy_return(self, _=0, data=None, **kwargs):
        flags = ['raise', 'logg', 'user', 'intern']
        flag = flags[_] if isinstance(_, int) else _
        if self.info.exec_code == 0:
            return self if data is None else data if _test_is_result(data) else self.ok(data=data, **kwargs)
        if flag == 'raise':
            raise ValueError(self.print(show=False))
        if flag == 'logg':
            from .. import get_logger
            get_logger().error(self.print(show=False))

        if flag == 'user':
            return self if data is None else data if _test_is_result(data) else self.default_user_error(data=data,
                                                                                                        **kwargs)
        if flag == 'intern':
            return self if data is None else data if _test_is_result(data) else self.default_internal_error(data=data,
                                                                                                            **kwargs)

        return self if data is None else data if _test_is_result(data) else self.custom_error(data=data, **kwargs)

    @property
    def bg_task(self):
        return self._task

__class_getitem__(item)

Enable Result[Type] syntax

Source code in toolboxv2/utils/system/types.py
643
644
645
646
647
648
649
650
651
652
653
def __class_getitem__(cls, item):
    """Enable Result[Type] syntax"""

    class TypedResult(cls):
        _generic_type = item

        def __init__(self, *args, **kwargs):
            super().__init__(*args, **kwargs)
            self._generic_type = item

    return TypedResult

binary(data, content_type='application/octet-stream', download_name=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a binary data response Result.

Source code in toolboxv2/utils/system/types.py
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
@classmethod
def binary(cls, data, content_type="application/octet-stream", download_name=None, info="OK",
           interface=ToolBoxInterfaces.remote):
    """Create a binary data response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    # Create a dictionary with binary data and metadata
    binary_data = {
        "data": data,
        "content_type": content_type,
        "filename": download_name
    }

    result = ToolBoxResult(
        data_to=interface,
        data=binary_data,
        data_info=f"Binary response: {download_name}" if download_name else "Binary response",
        data_type="binary"
    )

    return cls(error=error, info=info_obj, result=result)

cast_to(target_type)

Cast result to different type

Source code in toolboxv2/utils/system/types.py
738
739
740
741
742
743
744
745
746
747
748
def cast_to(self, target_type: Type[T]) -> 'Result[T]':
    """Cast result to different type"""
    new_result = Result(
        error=self.error,
        result=self.result,
        info=self.info,
        origin=self.origin,
        generic_type=target_type
    )
    new_result._generic_type = target_type
    return new_result

file(data, filename, content_type=None, info='OK', interface=ToolBoxInterfaces.remote) classmethod

Create a file download response Result.

Parameters:

Name Type Description Default
data

File data as bytes or base64 string

required
filename

Name of the file for download

required
content_type

MIME type of the file (auto-detected if None)

None
info

Response info text

'OK'
interface

Target interface

remote

Returns:

Type Description

Result object configured for file download

Source code in toolboxv2/utils/system/types.py
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
@classmethod
def file(cls, data, filename, content_type=None, info="OK", interface=ToolBoxInterfaces.remote):
    """Create a file download response Result.

    Args:
        data: File data as bytes or base64 string
        filename: Name of the file for download
        content_type: MIME type of the file (auto-detected if None)
        info: Response info text
        interface: Target interface

    Returns:
        Result object configured for file download
    """
    import base64
    import mimetypes

    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=200, help_text=info)

    # Auto-detect content type if not provided
    if content_type is None:
        content_type, _ = mimetypes.guess_type(filename)
        if content_type is None:
            content_type = "application/octet-stream"

    # Ensure data is base64 encoded string (as expected by Rust server)
    if isinstance(data, bytes):
        base64_data = base64.b64encode(data).decode('utf-8')
    elif isinstance(data, str):
        # Assume it's already base64 encoded
        base64_data = data
    else:
        raise ValueError("File data must be bytes or base64 string")

    result = ToolBoxResult(
        data_to=interface,
        data=base64_data,  # Rust expects base64 string for "file" type
        data_info=f"File download: {filename}",
        data_type="file"
    )

    return cls(error=error, info=info_obj, result=result)

get_type_info()

Get the generic type information

Source code in toolboxv2/utils/system/types.py
750
751
752
def get_type_info(self) -> Optional[Type]:
    """Get the generic type information"""
    return self._generic_type

is_typed()

Check if result has type information

Source code in toolboxv2/utils/system/types.py
754
755
756
def is_typed(self) -> bool:
    """Check if result has type information"""
    return self._generic_type is not None

json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create a JSON response Result.

Source code in toolboxv2/utils/system/types.py
970
971
972
973
974
975
976
977
978
979
980
981
982
983
@classmethod
def json(cls, data, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None):
    """Create a JSON response Result."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    return cls(error=error, info=info_obj, result=result)

redirect(url, status_code=302, info='Redirect', interface=ToolBoxInterfaces.remote) classmethod

Create a redirect response.

Source code in toolboxv2/utils/system/types.py
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
@classmethod
def redirect(cls, url, status_code=302, info="Redirect", interface=ToolBoxInterfaces.remote):
    """Create a redirect response."""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=url,
        data_info="Redirect response",
        data_type="redirect"
    )

    return cls(error=error, info=info_obj, result=result)

sse(stream_generator, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create an Server-Sent Events (SSE) streaming response Result.

Parameters:

Name Type Description Default
stream_generator Any

A source yielding individual data items. This can be an async generator, sync generator, iterable, or a single item. Each item will be formatted as an SSE event.

required
info str

Optional help text for the Result.

'OK'
interface ToolBoxInterfaces

Optional ToolBoxInterface to target.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional cleanup function to run when the stream ends or is cancelled.

None
#http_headers

Optional dictionary of custom HTTP headers for the SSE response.

required

Returns:

Type Description

A Result object configured for SSE streaming.

Source code in toolboxv2/utils/system/types.py
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
@classmethod
def sse(cls,
        stream_generator: Any,
        info: str = "OK",
        interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
        cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None,
        # http_headers: Optional[dict] = None # If we want to allow overriding default SSE HTTP headers
        ):
    """
    Create an Server-Sent Events (SSE) streaming response Result.

    Args:
        stream_generator: A source yielding individual data items. This can be an
                          async generator, sync generator, iterable, or a single item.
                          Each item will be formatted as an SSE event.
        info: Optional help text for the Result.
        interface: Optional ToolBoxInterface to target.
        cleanup_func: Optional cleanup function to run when the stream ends or is cancelled.
        #http_headers: Optional dictionary of custom HTTP headers for the SSE response.

    Returns:
        A Result object configured for SSE streaming.
    """
    # Result.stream will handle calling SSEGenerator.create_sse_stream
    # and setting appropriate default headers for SSE when content_type is "text/event-stream".
    return cls.stream(
        stream_generator=stream_generator,
        content_type="text/event-stream",
        # headers=http_headers, # Pass if we add http_headers param
        info=info,
        interface=interface,
        cleanup_func=cleanup_func
    )

stream(stream_generator, content_type='text/event-stream', headers=None, info='OK', interface=ToolBoxInterfaces.remote, cleanup_func=None) classmethod

Create a streaming response Result. Handles SSE and other stream types.

Parameters:

Name Type Description Default
stream_generator Any

Any stream source (async generator, sync generator, iterable, or single item).

required
content_type str

Content-Type header (default: text/event-stream for SSE).

'text/event-stream'
headers dict | None

Additional HTTP headers for the response.

None
info str

Help text for the result.

'OK'
interface ToolBoxInterfaces

Interface to send data to.

remote
cleanup_func Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None

Optional function for cleanup.

None

Returns:

Type Description

A Result object configured for streaming.

Source code in toolboxv2/utils/system/types.py
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
@classmethod
def stream(cls,
           stream_generator: Any,  # Renamed from source for clarity
           content_type: str = "text/event-stream",  # Default to SSE
           headers: dict | None = None,
           info: str = "OK",
           interface: ToolBoxInterfaces = ToolBoxInterfaces.remote,
           cleanup_func: Callable[[], None] | Callable[[], T] | Callable[[], AsyncGenerator[T, None]] | None = None):
    """
    Create a streaming response Result. Handles SSE and other stream types.

    Args:
        stream_generator: Any stream source (async generator, sync generator, iterable, or single item).
        content_type: Content-Type header (default: text/event-stream for SSE).
        headers: Additional HTTP headers for the response.
        info: Help text for the result.
        interface: Interface to send data to.
        cleanup_func: Optional function for cleanup.

    Returns:
        A Result object configured for streaming.
    """
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)

    final_generator: AsyncGenerator[str, None]

    if content_type == "text/event-stream":
        # For SSE, always use SSEGenerator.create_sse_stream to wrap the source.
        # SSEGenerator.create_sse_stream handles various types of stream_generator internally.
        final_generator = SSEGenerator.create_sse_stream(source=stream_generator, cleanup_func=cleanup_func)

        # Standard SSE headers for the HTTP response itself
        # These will be stored in the Result object. Rust side decides how to use them.
        standard_sse_headers = {
            "Cache-Control": "no-cache",  # SSE specific
            "Connection": "keep-alive",  # SSE specific
            "X-Accel-Buffering": "no",  # Useful for proxies with SSE
            # Content-Type is implicitly text/event-stream, will be in streaming_data below
        }
        all_response_headers = standard_sse_headers.copy()
        if headers:
            all_response_headers.update(headers)
    else:
        # For non-SSE streams.
        # If stream_generator is sync, wrap it to be async.
        # If already async or single item, it will be handled.
        # Rust's stream_generator in ToolboxClient seems to handle both sync/async Python generators.
        # For consistency with how SSEGenerator does it, we can wrap sync ones.
        if inspect.isgenerator(stream_generator) or \
            (not isinstance(stream_generator, str) and hasattr(stream_generator, '__iter__')):
            final_generator = SSEGenerator.wrap_sync_generator(stream_generator)  # Simple async wrapper
        elif inspect.isasyncgen(stream_generator):
            final_generator = stream_generator
        else:  # Single item or string
            async def _single_item_gen():
                yield stream_generator

            final_generator = _single_item_gen()
        all_response_headers = headers if headers else {}

    # Prepare streaming data to be stored in the Result object
    streaming_data = {
        "type": "stream",  # Indicator for Rust side
        "generator": final_generator,
        "content_type": content_type,  # Let Rust know the intended content type
        "headers": all_response_headers  # Intended HTTP headers for the overall response
    }

    result_payload = ToolBoxResult(
        data_to=interface,
        data=streaming_data,
        data_info="Streaming response" if content_type != "text/event-stream" else "SSE Event Stream",
        data_type="stream"  # Generic type for Rust to identify it needs to stream from 'generator'
    )

    return cls(error=error, info=info_obj, result=result_payload)

text(text_data, content_type='text/plain', exec_code=None, status=200, info='OK', interface=ToolBoxInterfaces.remote, headers=None) classmethod

Create a text response Result with specific content type.

Source code in toolboxv2/utils/system/types.py
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
@classmethod
def text(cls, text_data, content_type="text/plain",exec_code=None,status=200, info="OK", interface=ToolBoxInterfaces.remote, headers=None):
    """Create a text response Result with specific content type."""
    if headers is not None:
        return cls.html(text_data, status= exec_code or status, info=info, headers=headers)
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=exec_code or status, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=text_data,
        data_info="Text response",
        data_type=content_type
    )

    return cls(error=error, info=info_obj, result=result)

typed_aget(key=None, default=None) async

Async get data with type validation

Source code in toolboxv2/utils/system/types.py
667
668
669
670
671
672
673
674
675
676
async def typed_aget(self, key=None, default=None) -> T:
    """Async get data with type validation"""
    data = await self.aget(key, default)

    if self._generic_type and data is not None:
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_get(key=None, default=None)

Get data with type validation

Source code in toolboxv2/utils/system/types.py
655
656
657
658
659
660
661
662
663
664
665
def typed_get(self, key=None, default=None) -> T:
    """Get data with type validation"""
    data = self.get(key, default)

    if self._generic_type and data is not None:
        # Validate type matches generic parameter
        if not self._validate_type(data, self._generic_type):
            from toolboxv2 import get_logger
            get_logger().warning(f"Type mismatch: expected {self._generic_type}, got {type(data)}")

    return data

typed_json(data, info='OK', interface=ToolBoxInterfaces.remote, exec_code=0, status_code=None) classmethod

Create JSON result with type information

Source code in toolboxv2/utils/system/types.py
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
@classmethod
def typed_json(cls, data: T, info="OK", interface=ToolBoxInterfaces.remote, exec_code=0,
               status_code=None) -> 'Result[T]':
    """Create JSON result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=status_code or exec_code, help_text=info)

    result = ToolBoxResult(
        data_to=interface,
        data=data,
        data_info="JSON response",
        data_type="json"
    )

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

typed_ok(data, data_info='', info='OK', interface=ToolBoxInterfaces.native) classmethod

Create OK result with type information

Source code in toolboxv2/utils/system/types.py
705
706
707
708
709
710
711
712
713
714
715
716
@classmethod
def typed_ok(cls, data: T, data_info="", info="OK", interface=ToolBoxInterfaces.native) -> 'Result[T]':
    """Create OK result with type information"""
    error = ToolBoxError.none
    info_obj = ToolBoxInfo(exec_code=0, help_text=info)
    result = ToolBoxResult(data_to=interface, data=data, data_info=data_info, data_type=type(data).__name__)

    instance = cls(error=error, info=info_obj, result=result)
    if hasattr(cls, '_generic_type'):
        instance._generic_type = cls._generic_type

    return instance

toolboxv2.ApiResult

Bases: BaseModel

Source code in toolboxv2/utils/system/types.py
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
class ApiResult(BaseModel):
    error: None | str= None
    origin: Any | None
    result: ToolBoxResultBM | None = None
    info: ToolBoxInfoBM | None

    def as_result(self):
        return Result(
            error=self.error.value if isinstance(self.error, Enum) else self.error,
            result=ToolBoxResult(
                data_to=self.result.data_to.value if isinstance(self.result.data_to, Enum) else self.result.data_to,
                data_info=self.result.data_info,
                data=self.result.data,
                data_type=self.result.data_type
            ) if self.result else None,
            info=ToolBoxInfo(
                exec_code=self.info.exec_code,
                help_text=self.info.help_text
            ) if self.info else None,
            origin=self.origin
        )

    def to_api_result(self):
        return self

    def print(self, *args, **kwargs):
        res = self.as_result().print(*args, **kwargs)
        if not isinstance(res, str):
            res = res.to_api_result()
        return res

toolboxv2.RequestData

Main class representing the complete request data structure.

Source code in toolboxv2/utils/system/types.py
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
@dataclass
class RequestData:
    """Main class representing the complete request data structure."""
    request: Request
    session: Session
    session_id: str

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
        """Create a RequestData instance from a dictionary."""
        return cls(
            request=Request.from_dict(data.get('request', {})),
            session=Session.from_dict(data.get('session', {})),
            session_id=data.get('session_id', '')
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert the RequestData object back to a dictionary."""
        return {
            'request': self.request.to_dict(),
            'session': self.session.to_dict(),
            'session_id': self.session_id
        }

    def __getattr__(self, name: str) -> Any:
        """Delegate unknown attributes to the `request` object."""
        # Nur wenn das Attribut nicht direkt in RequestData existiert
        # und auch nicht `session` oder `session_id` ist
        if hasattr(self.request, name):
            return getattr(self.request, name)
        raise AttributeError(f"'RequestData' object has no attribute '{name}'")

    @classmethod
    def moc(cls):
        return cls(
            request=Request.from_dict({
                'content_type': 'application/x-www-form-urlencoded',
                'headers': {
                    'accept': '*/*',
                    'accept-encoding': 'gzip, deflate, br, zstd',
                    'accept-language': 'de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7',
                    'connection': 'keep-alive',
                    'content-length': '107',
                    'content-type': 'application/x-www-form-urlencoded',
                    'cookie': 'session=abc123',
                    'host': 'localhost:8080',
                    'hx-current-url': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'hx-request': 'true',
                    'hx-target': 'estimates-guest_1fc2c9',
                    'hx-trigger': 'config-form-guest_1fc2c9',
                    'origin': 'http://localhost:8080',
                    'referer': 'http://localhost:8080/api/TruthSeeker/get_main_ui',
                    'sec-ch-ua': '"Chromium";v="134", "Not:A-Brand";v="24", "Google Chrome";v="134"',
                    'sec-ch-ua-mobile': '?0',
                    'sec-ch-ua-platform': '"Windows"',
                    'sec-fetch-dest': 'empty',
                    'sec-fetch-mode': 'cors',
                    'sec-fetch-site': 'same-origin',
                    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
                },
                'method': 'POST',
                'path': '/api/TruthSeeker/update_estimates',
                'query_params': {},
                'form_data': {
                    'param1': 'value1',
                    'param2': 'value2'
                }
            }),
            session=Session.from_dict({
                'SiID': '29a2e258e18252e2afd5ff943523f09c82f1bb9adfe382a6f33fc6a8381de898',
                'level': '1',
                'spec': '74eed1c8de06886842e235486c3c2fd6bcd60586998ac5beb87f13c0d1750e1d',
                'user_name': 'root',
                'custom_field': 'custom_value'
            }),
            session_id='0x29dd1ac0d1e30d3f'
        )

__getattr__(name)

Delegate unknown attributes to the request object.

Source code in toolboxv2/utils/system/types.py
326
327
328
329
330
331
332
def __getattr__(self, name: str) -> Any:
    """Delegate unknown attributes to the `request` object."""
    # Nur wenn das Attribut nicht direkt in RequestData existiert
    # und auch nicht `session` oder `session_id` ist
    if hasattr(self.request, name):
        return getattr(self.request, name)
    raise AttributeError(f"'RequestData' object has no attribute '{name}'")

from_dict(data) classmethod

Create a RequestData instance from a dictionary.

Source code in toolboxv2/utils/system/types.py
309
310
311
312
313
314
315
316
@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'RequestData':
    """Create a RequestData instance from a dictionary."""
    return cls(
        request=Request.from_dict(data.get('request', {})),
        session=Session.from_dict(data.get('session', {})),
        session_id=data.get('session_id', '')
    )

to_dict()

Convert the RequestData object back to a dictionary.

Source code in toolboxv2/utils/system/types.py
318
319
320
321
322
323
324
def to_dict(self) -> dict[str, Any]:
    """Convert the RequestData object back to a dictionary."""
    return {
        'request': self.request.to_dict(),
        'session': self.session.to_dict(),
        'session_id': self.session_id
    }

Security

toolboxv2.Code

Source code in toolboxv2/utils/security/cryp.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
class Code:

    @staticmethod
    def DK():
        return DEVICE_KEY

    @staticmethod
    def generate_random_string(length: int) -> str:
        """
        Generiert eine zufällige Zeichenkette der angegebenen Länge.

        Args:
            length (int): Die Länge der zu generierenden Zeichenkette.

        Returns:
            str: Die generierte Zeichenkette.
        """
        return secrets.token_urlsafe(length)

    def decode_code(self, encrypted_data, key=None):

        if not isinstance(encrypted_data, str):
            encrypted_data = str(encrypted_data)

        if key is None:
            key = DEVICE_KEY()

        return self.decrypt_symmetric(encrypted_data, key)

    def encode_code(self, data, key=None):

        if not isinstance(data, str):
            data = str(data)

        if key is None:
            key = DEVICE_KEY()

        return self.encrypt_symmetric(data, key)

    @staticmethod
    def generate_seed() -> int:
        """
        Erzeugt eine zufällige Zahl als Seed.

        Returns:
            int: Eine zufällige Zahl.
        """
        return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

    @staticmethod
    def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
        """
        Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

        Args:
            text (str): Der zu hashende Text.
            salt (str): Der Salt-Wert.
            pepper (str): Der Pepper-Wert.
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            str: Der resultierende Hash-Wert.
        """
        return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

    @staticmethod
    def generate_symmetric_key(as_str=True) -> str or bytes:
        """
        Generiert einen Schlüssel für die symmetrische Verschlüsselung.

        Returns:
            str: Der generierte Schlüssel.
        """
        key = Fernet.generate_key()
        if as_str:
            key = key.decode()
        return key

    @staticmethod
    def encrypt_symmetric(text: str or bytes, key: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.

        Returns:
            str: Der verschlüsselte Text.
        """
        if isinstance(text, str):
            text = text.encode()

        try:
            fernet = Fernet(key.encode())
            return fernet.encrypt(text).decode()
        except Exception as e:
            get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
            return "Error encrypt"

    @staticmethod
    def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
        """
        Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

        Args:
            encrypted_text (str): Der zu entschlüsselnde Text.
            key (str): Der symmetrische Schlüssel.
            to_str (bool): default true returns str if false returns bytes
        Returns:
            str: Der entschlüsselte Text.
        """

        if isinstance(key, str):
            key = key.encode()

        #try:
        fernet = Fernet(key)
        text_b = fernet.decrypt(encrypted_text)
        if not to_str:
            return text_b
        return text_b.decode()
        # except Exception as e:
        #     get_logger().error(f"Error decrypt_symmetric {e}")
        #     if not mute:
        #         raise e
        #     if not to_str:
        #         return f"Error decoding".encode()
        #     return f"Error decoding"

    @staticmethod
    def generate_asymmetric_keys() -> (str, str):
        """
        Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

        Args:
            seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
        """
        private_key = rsa.generate_private_key(
            public_exponent=65537,
            key_size=2048 * 3,
        )
        public_key = private_key.public_key()

        # Serialisieren der Schlüssel
        pem_private_key = private_key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.PKCS8,
            encryption_algorithm=serialization.NoEncryption()
        ).decode()

        pem_public_key = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        ).decode()

        return pem_public_key, pem_private_key

    @staticmethod
    def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
        """
        Speichert die generierten Schlüssel in separate Dateien.
        Der private Schlüssel wird mit dem Device Key verschlüsselt.

        Args:
            public_key (str): Der öffentliche Schlüssel im PEM-Format
            private_key (str): Der private Schlüssel im PEM-Format
            directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
        """
        # Erstelle das Verzeichnis, falls es nicht existiert
        os.makedirs(directory, exist_ok=True)

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Verschlüssele den privaten Schlüssel mit dem Device Key
        encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

        # Speichere den öffentlichen Schlüssel
        public_key_path = os.path.join(directory, "public_key.pem")
        with open(public_key_path, "w") as f:
            f.write(public_key)

        # Speichere den verschlüsselten privaten Schlüssel
        private_key_path = os.path.join(directory, "private_key.pem")
        with open(private_key_path, "w") as f:
            f.write(encrypted_private_key)

        print("Saved keys in ", public_key_path)

    @staticmethod
    def load_keys_from_files(directory: str = "keys") -> (str, str):
        """
        Lädt die Schlüssel aus den Dateien.
        Der private Schlüssel wird mit dem Device Key entschlüsselt.

        Args:
            directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

        Returns:
            (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

        Raises:
            FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
        """
        # Pfade zu den Schlüsseldateien
        public_key_path = os.path.join(directory, "public_key.pem")
        private_key_path = os.path.join(directory, "private_key.pem")

        # Prüfe ob die Dateien existieren
        if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
            return "", ""

        # Hole den Device Key
        device_key = DEVICE_KEY()

        # Lade den öffentlichen Schlüssel
        with open(public_key_path) as f:
            public_key = f.read()

        # Lade und entschlüssele den privaten Schlüssel
        with open(private_key_path) as f:
            encrypted_private_key = f.read()
            private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

        return public_key, private_key

    @staticmethod
    def encrypt_asymmetric(text: str, public_key_str: str) -> str:
        """
        Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

        Args:
            text (str): Der zu verschlüsselnde Text.
            public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

        Returns:
            str: Der verschlüsselte Text.
        """
        # try:
        #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        #  except Exception as e:
        #     get_logger().error(f"Error encrypt_asymmetric {e}")
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            encrypted = public_key.encrypt(
                text.encode(),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return encrypted.hex()
        except Exception as e:
            get_logger().error(f"Error encrypt_asymmetric {e}")
            return "Invalid"

    @staticmethod
    def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
        """
        Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

        Args:
            encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
            private_key_str (str): Der private Schlüssel als String.

        Returns:
            str: Der entschlüsselte Text.
        """
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            decrypted = private_key.decrypt(
                bytes.fromhex(encrypted_text_hex),
                padding.OAEP(
                    mgf=padding.MGF1(algorithm=hashes.SHA512()),
                    algorithm=hashes.SHA512(),
                    label=None
                )
            )
            return decrypted.decode()

        except Exception as e:
            get_logger().error(f"Error decrypt_asymmetric {e}")
        return "Invalid"

    @staticmethod
    def verify_signature(signature: str or bytes, message: str or bytes, public_key_str: str,
                         salt_length=padding.PSS.MAX_LENGTH) -> bool:
        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                padding=padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                algorithm=hashes.SHA512()
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def verify_signature_web_algo(signature: str or bytes, message: str or bytes, public_key_str: str,
                                  algo: int = -512) -> bool:
        signature_algorithm = ECDSA(hashes.SHA512())
        if algo != -512:
            signature_algorithm = ECDSA(hashes.SHA256())

        if isinstance(signature, str):
            signature = signature.encode()
        if isinstance(message, str):
            message = message.encode()
        try:
            public_key = serialization.load_pem_public_key(public_key_str.encode())
            public_key.verify(
                signature=signature,
                data=message,
                # padding=padding.PSS(
                #    mgf=padding.MGF1(hashes.SHA512()),
                #    salt_length=padding.PSS.MAX_LENGTH
                # ),
                signature_algorithm=signature_algorithm
            )
            return True
        except:
            pass
        return False

    @staticmethod
    def create_signature(message: str, private_key_str: str, salt_length=padding.PSS.MAX_LENGTH,
                         row=False) -> str or bytes:
        try:
            private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
            signature = private_key.sign(
                message.encode(),
                padding.PSS(
                    mgf=padding.MGF1(hashes.SHA512()),
                    salt_length=salt_length
                ),
                hashes.SHA512()
            )
            if row:
                return signature
            return base64.b64encode(signature).decode()
        except Exception as e:
            get_logger().error(f"Error create_signature {e}")
            print(e)
        return "Invalid Key"

    @staticmethod
    def pem_to_public_key(pem_key: str):
        """
        Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

        Args:
            pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

        Returns:
            PublicKey: Das PublicKey-Objekt.
        """
        public_key = serialization.load_pem_public_key(pem_key.encode())
        return public_key

    @staticmethod
    def public_key_to_pem(public_key: RSAPublicKey):
        """
        Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

        Args:
            public_key (PublicKey): Das PublicKey-Objekt.

        Returns:
            str: Der PEM-kodierte öffentliche Schlüssel.
        """
        pem = public_key.public_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PublicFormat.SubjectPublicKeyInfo
        )
        return pem.decode()

decrypt_asymmetric(encrypted_text_hex, private_key_str) staticmethod

Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

Parameters:

Name Type Description Default
encrypted_text_hex str

Der verschlüsselte Text als Hex-String.

required
private_key_str str

Der private Schlüssel als String.

required

Returns:

Name Type Description
str str

Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
@staticmethod
def decrypt_asymmetric(encrypted_text_hex: str, private_key_str: str) -> str:
    """
    Entschlüsselt einen Text mit einem gegebenen privaten Schlüssel.

    Args:
        encrypted_text_hex (str): Der verschlüsselte Text als Hex-String.
        private_key_str (str): Der private Schlüssel als String.

    Returns:
        str: Der entschlüsselte Text.
    """
    try:
        private_key = serialization.load_pem_private_key(private_key_str.encode(), password=None)
        decrypted = private_key.decrypt(
            bytes.fromhex(encrypted_text_hex),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return decrypted.decode()

    except Exception as e:
        get_logger().error(f"Error decrypt_asymmetric {e}")
    return "Invalid"

decrypt_symmetric(encrypted_text, key, to_str=True, mute=False) staticmethod

Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
encrypted_text str

Der zu entschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required
to_str bool

default true returns str if false returns bytes

True

Returns: str: Der entschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
@staticmethod
def decrypt_symmetric(encrypted_text: str, key: str, to_str=True, mute=False) -> str or bytes:
    """
    Entschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        encrypted_text (str): Der zu entschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.
        to_str (bool): default true returns str if false returns bytes
    Returns:
        str: Der entschlüsselte Text.
    """

    if isinstance(key, str):
        key = key.encode()

    #try:
    fernet = Fernet(key)
    text_b = fernet.decrypt(encrypted_text)
    if not to_str:
        return text_b
    return text_b.decode()

encrypt_asymmetric(text, public_key_str) staticmethod

Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
public_key_str str

Der öffentliche Schlüssel als String oder im pem format.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
@staticmethod
def encrypt_asymmetric(text: str, public_key_str: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen öffentlichen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        public_key_str (str): Der öffentliche Schlüssel als String oder im pem format.

    Returns:
        str: Der verschlüsselte Text.
    """
    # try:
    #    public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
    #  except Exception as e:
    #     get_logger().error(f"Error encrypt_asymmetric {e}")
    try:
        public_key: RSAPublicKey = serialization.load_pem_public_key(public_key_str.encode())
        encrypted = public_key.encrypt(
            text.encode(),
            padding.OAEP(
                mgf=padding.MGF1(algorithm=hashes.SHA512()),
                algorithm=hashes.SHA512(),
                label=None
            )
        )
        return encrypted.hex()
    except Exception as e:
        get_logger().error(f"Error encrypt_asymmetric {e}")
        return "Invalid"

encrypt_symmetric(text, key) staticmethod

Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

Parameters:

Name Type Description Default
text str

Der zu verschlüsselnde Text.

required
key str

Der symmetrische Schlüssel.

required

Returns:

Name Type Description
str str

Der verschlüsselte Text.

Source code in toolboxv2/utils/security/cryp.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
@staticmethod
def encrypt_symmetric(text: str or bytes, key: str) -> str:
    """
    Verschlüsselt einen Text mit einem gegebenen symmetrischen Schlüssel.

    Args:
        text (str): Der zu verschlüsselnde Text.
        key (str): Der symmetrische Schlüssel.

    Returns:
        str: Der verschlüsselte Text.
    """
    if isinstance(text, str):
        text = text.encode()

    try:
        fernet = Fernet(key.encode())
        return fernet.encrypt(text).decode()
    except Exception as e:
        get_logger().error(f"Error encrypt_symmetric #{str(e)}#")
        return "Error encrypt"

generate_asymmetric_keys() staticmethod

Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

Parameters:

Name Type Description Default
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
@staticmethod
def generate_asymmetric_keys() -> (str, str):
    """
    Generiert ein Paar von öffentlichen und privaten Schlüsseln für die asymmetrische Verschlüsselung.

    Args:
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel.
    """
    private_key = rsa.generate_private_key(
        public_exponent=65537,
        key_size=2048 * 3,
    )
    public_key = private_key.public_key()

    # Serialisieren der Schlüssel
    pem_private_key = private_key.private_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PrivateFormat.PKCS8,
        encryption_algorithm=serialization.NoEncryption()
    ).decode()

    pem_public_key = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    ).decode()

    return pem_public_key, pem_private_key

generate_random_string(length) staticmethod

Generiert eine zufällige Zeichenkette der angegebenen Länge.

Parameters:

Name Type Description Default
length int

Die Länge der zu generierenden Zeichenkette.

required

Returns:

Name Type Description
str str

Die generierte Zeichenkette.

Source code in toolboxv2/utils/security/cryp.py
81
82
83
84
85
86
87
88
89
90
91
92
@staticmethod
def generate_random_string(length: int) -> str:
    """
    Generiert eine zufällige Zeichenkette der angegebenen Länge.

    Args:
        length (int): Die Länge der zu generierenden Zeichenkette.

    Returns:
        str: Die generierte Zeichenkette.
    """
    return secrets.token_urlsafe(length)

generate_seed() staticmethod

Erzeugt eine zufällige Zahl als Seed.

Returns:

Name Type Description
int int

Eine zufällige Zahl.

Source code in toolboxv2/utils/security/cryp.py
114
115
116
117
118
119
120
121
122
@staticmethod
def generate_seed() -> int:
    """
    Erzeugt eine zufällige Zahl als Seed.

    Returns:
        int: Eine zufällige Zahl.
    """
    return random.randint(2 ** 32 - 1, 2 ** 64 - 1)

generate_symmetric_key(as_str=True) staticmethod

Generiert einen Schlüssel für die symmetrische Verschlüsselung.

Returns:

Name Type Description
str str or bytes

Der generierte Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
140
141
142
143
144
145
146
147
148
149
150
151
@staticmethod
def generate_symmetric_key(as_str=True) -> str or bytes:
    """
    Generiert einen Schlüssel für die symmetrische Verschlüsselung.

    Returns:
        str: Der generierte Schlüssel.
    """
    key = Fernet.generate_key()
    if as_str:
        key = key.decode()
    return key

load_keys_from_files(directory='keys') staticmethod

Lädt die Schlüssel aus den Dateien. Der private Schlüssel wird mit dem Device Key entschlüsselt.

Parameters:

Name Type Description Default
directory str

Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

'keys'

Returns:

Type Description
(str, str)

Ein Tupel aus öffentlichem und privatem Schlüssel

Raises:

Type Description
FileNotFoundError

Wenn die Schlüsseldateien nicht gefunden werden können

Source code in toolboxv2/utils/security/cryp.py
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
@staticmethod
def load_keys_from_files(directory: str = "keys") -> (str, str):
    """
    Lädt die Schlüssel aus den Dateien.
    Der private Schlüssel wird mit dem Device Key entschlüsselt.

    Args:
        directory (str): Das Verzeichnis, aus dem die Schlüssel geladen werden sollen

    Returns:
        (str, str): Ein Tupel aus öffentlichem und privatem Schlüssel

    Raises:
        FileNotFoundError: Wenn die Schlüsseldateien nicht gefunden werden können
    """
    # Pfade zu den Schlüsseldateien
    public_key_path = os.path.join(directory, "public_key.pem")
    private_key_path = os.path.join(directory, "private_key.pem")

    # Prüfe ob die Dateien existieren
    if not os.path.exists(public_key_path) or not os.path.exists(private_key_path):
        return "", ""

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Lade den öffentlichen Schlüssel
    with open(public_key_path) as f:
        public_key = f.read()

    # Lade und entschlüssele den privaten Schlüssel
    with open(private_key_path) as f:
        encrypted_private_key = f.read()
        private_key = Code.decrypt_symmetric(encrypted_private_key, device_key)

    return public_key, private_key

one_way_hash(text, salt='', pepper='') staticmethod

Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

Parameters:

Name Type Description Default
text str

Der zu hashende Text.

required
salt str

Der Salt-Wert.

''
pepper str

Der Pepper-Wert.

''
seed int

Ein optionaler Seed-Wert. Standardmäßig None.

required

Returns:

Name Type Description
str str

Der resultierende Hash-Wert.

Source code in toolboxv2/utils/security/cryp.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
@staticmethod
def one_way_hash(text: str, salt: str = '', pepper: str = '') -> str:
    """
    Erzeugt einen Hash eines gegebenen Textes mit Salt, Pepper und optional einem Seed.

    Args:
        text (str): Der zu hashende Text.
        salt (str): Der Salt-Wert.
        pepper (str): Der Pepper-Wert.
        seed (int, optional): Ein optionaler Seed-Wert. Standardmäßig None.

    Returns:
        str: Der resultierende Hash-Wert.
    """
    return hashlib.sha256((salt + text + pepper).encode()).hexdigest()

pem_to_public_key(pem_key) staticmethod

Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

Parameters:

Name Type Description Default
pem_key str

Der PEM-kodierte öffentliche Schlüssel.

required

Returns:

Name Type Description
PublicKey

Das PublicKey-Objekt.

Source code in toolboxv2/utils/security/cryp.py
435
436
437
438
439
440
441
442
443
444
445
446
447
@staticmethod
def pem_to_public_key(pem_key: str):
    """
    Konvertiert einen PEM-kodierten öffentlichen Schlüssel in ein PublicKey-Objekt.

    Args:
        pem_key (str): Der PEM-kodierte öffentliche Schlüssel.

    Returns:
        PublicKey: Das PublicKey-Objekt.
    """
    public_key = serialization.load_pem_public_key(pem_key.encode())
    return public_key

public_key_to_pem(public_key) staticmethod

Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

Parameters:

Name Type Description Default
public_key PublicKey

Das PublicKey-Objekt.

required

Returns:

Name Type Description
str

Der PEM-kodierte öffentliche Schlüssel.

Source code in toolboxv2/utils/security/cryp.py
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
@staticmethod
def public_key_to_pem(public_key: RSAPublicKey):
    """
    Konvertiert ein PublicKey-Objekt in einen PEM-kodierten String.

    Args:
        public_key (PublicKey): Das PublicKey-Objekt.

    Returns:
        str: Der PEM-kodierte öffentliche Schlüssel.
    """
    pem = public_key.public_bytes(
        encoding=serialization.Encoding.PEM,
        format=serialization.PublicFormat.SubjectPublicKeyInfo
    )
    return pem.decode()

save_keys_to_files(public_key, private_key, directory='keys') staticmethod

Speichert die generierten Schlüssel in separate Dateien. Der private Schlüssel wird mit dem Device Key verschlüsselt.

Parameters:

Name Type Description Default
public_key str

Der öffentliche Schlüssel im PEM-Format

required
private_key str

Der private Schlüssel im PEM-Format

required
directory str

Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen

'keys'
Source code in toolboxv2/utils/security/cryp.py
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
@staticmethod
def save_keys_to_files(public_key: str, private_key: str, directory: str = "keys") -> None:
    """
    Speichert die generierten Schlüssel in separate Dateien.
    Der private Schlüssel wird mit dem Device Key verschlüsselt.

    Args:
        public_key (str): Der öffentliche Schlüssel im PEM-Format
        private_key (str): Der private Schlüssel im PEM-Format
        directory (str): Das Verzeichnis, in dem die Schlüssel gespeichert werden sollen
    """
    # Erstelle das Verzeichnis, falls es nicht existiert
    os.makedirs(directory, exist_ok=True)

    # Hole den Device Key
    device_key = DEVICE_KEY()

    # Verschlüssele den privaten Schlüssel mit dem Device Key
    encrypted_private_key = Code.encrypt_symmetric(private_key, device_key)

    # Speichere den öffentlichen Schlüssel
    public_key_path = os.path.join(directory, "public_key.pem")
    with open(public_key_path, "w") as f:
        f.write(public_key)

    # Speichere den verschlüsselten privaten Schlüssel
    private_key_path = os.path.join(directory, "private_key.pem")
    with open(private_key_path, "w") as f:
        f.write(encrypted_private_key)

    print("Saved keys in ", public_key_path)

Modules & Flows

toolboxv2.mods

Canvas

Tools

Bases: MainTool

Source code in toolboxv2/mods/Canvas.py
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
class Tools(MainTool):  # Removed EventManager for simplicity, as it was causing the issue. Direct SSE is better here.
    def __init__(self, app: App):
        self.name = MOD_NAME
        self.version = VERSION
        self.color = "GREEN"
        self.tools_dict = {"name": MOD_NAME, "Version": self.show_version}

        # Canvas specific state
        self.live_canvas_sessions: dict[str, list[asyncio.Queue]] = defaultdict(list)
        self.active_user_previews: dict[str, dict[str, Any]] = defaultdict(dict)
        self.previews_lock = asyncio.Lock()

        MainTool.__init__(self, load=on_start, v=self.version, tool=self.tools_dict, name=self.name,
                          color=self.color, app=app)
        self.app.logger.info(f"Canvas Tools (v{self.version}) initialized for app {self.app.id}.")

    @property
    def db_mod(self):
        db = self.app.get_mod("DB", spec=Name)
        if db.mode.value != "CLUSTER_BLOB":
            db.edit_cli("CB")
        return db

    def _broadcast_to_canvas_listeners(self, canvas_id: str, event_type: str, data: dict[str, Any],
                                       originator_user_id: str | None = None):
        """
        Creates a broadcast coroutine and submits it to the app's dedicated
        async manager to be run in the background.
        This is now a non-blocking fire-and-forget operation.
        """

        async def broadcast_coro():
            if canvas_id not in self.live_canvas_sessions:
                return

            message_obj = {
                "event": event_type,
                "data": json.dumps({
                    "canvas_id": canvas_id,
                    "originator_user_id": originator_user_id,
                    **data
                })
            }

            listeners = list(self.live_canvas_sessions.get(canvas_id, []))

            for q in listeners:
                try:
                    # Non-blocking put. If the queue is full, the client is lagging,
                    # and it's better to drop a message than to block the server.
                    q.put_nowait(message_obj)
                except asyncio.QueueFull:
                    self.app.logger.warning(
                        f"SSE queue full for canvas {canvas_id}. Message '{event_type}' dropped for one client.")
                except Exception as e:
                    self.app.logger.error(f"Error putting message on SSE queue: {e}")

        # Use the app's robust background runner to execute immediately and not block the caller.
        self.app.run_bg_task(broadcast_coro)

    def show_version(self):
        self.app.logger.info(f"{self.name} Version: {self.version}")
        return self.version

    async def _get_user_specific_db_key(self, request: RequestData, base_key: str) -> str | None:
        # This logic is correct and can remain as is.

        user = await get_user_from_request(self.app, request)
        if user and user.uid:
            return f"{base_key}_{user.uid}"
        self.print("ok")
        # Fallback for public/guest access if you want to support it
        return f"{base_key}_public"

handle_send_canvas_action(app, request, data) async

Handles incremental, real-time actions from clients (e.g., adding an element). It persists the change to the database and then broadcasts it to all live listeners.

Source code in toolboxv2/mods/Canvas.py
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
@export(mod_name=MOD_NAME, api=True, version=VERSION, name="send_canvas_action", api_methods=['POST'],
        request_as_kwarg=True)
async def handle_send_canvas_action(app: App, request: RequestData, data: dict[str, Any]):
    """
    Handles incremental, real-time actions from clients (e.g., adding an element).
    It persists the change to the database and then broadcasts it to all live listeners.
    """
    canvas_tool = app.get_mod(MOD_NAME)
    if not canvas_tool or not canvas_tool.db_mod:
        return Result.default_internal_error("Canvas module or DB not loaded.")

    if not data:
        return Result.default_user_error("Request data is missing.", 400)

    canvas_id = data.get("canvas_id")
    action_type = data.get("action_type")
    action_payload = data.get("payload")
    user_id = data.get("user_id")

    if not all([canvas_id, action_type, user_id]) or action_payload is None:
        return Result.default_user_error("Request missing required fields.", 400)

    # --- Flow 1: Ephemeral 'preview' actions that DO NOT get persisted ---
    if action_type in ["preview_update", "preview_clear"]:
        sse_event_type = "user_preview_update" if action_type == "preview_update" else "clear_user_preview"
        sse_data = {"user_id": user_id}

        async with canvas_tool.previews_lock:
            if action_type == "preview_update":
                canvas_tool.active_user_previews[canvas_id][user_id] = action_payload
                sse_data["preview_data"] = action_payload
            elif user_id in canvas_tool.active_user_previews.get(canvas_id, {}):
                del canvas_tool.active_user_previews[canvas_id][user_id]

        # MODIFICATION: Call the non-blocking broadcast method. This returns immediately.
        canvas_tool._broadcast_to_canvas_listeners(
            canvas_id=canvas_id, event_type=sse_event_type,
            data=sse_data, originator_user_id=user_id
        )
        return Result.ok(info=f"'{action_type}' broadcasted.")

    # --- Flow 2: Persistent actions that modify the canvas state ---
    if action_type not in ["element_add", "element_update", "element_remove"]:
        return Result.default_user_error(f"Unknown persistent action_type: {action_type}", 400)

    # Load the full, current session state from the database
    user_db_key_base = await canvas_tool._get_user_specific_db_key(request, SESSION_DATA_PREFIX)
    session_db_key = f"{user_db_key_base}_{canvas_id}"
    try:
        db_result = canvas_tool.db_mod.get(session_db_key)
        if not db_result or db_result.is_error() or not db_result.get():
            return Result.default_user_error("Canvas session not found in database.", 404)

        session_data_str = db_result.get()[0] if isinstance(db_result.get(), list) else db_result.get()
        session_data = IdeaSessionData.model_validate_json(session_data_str)
    except Exception as e:
        app.logger.error(f"DB Load/Parse failed for C:{canvas_id}. Error: {e}", exc_info=True)
        return Result.default_internal_error("Could not load canvas data to apply changes.")

    # Apply the action to the in-memory Pydantic object
    if action_type == "element_add":
        session_data.canvas_elements.append(CanvasElement(**action_payload))
    elif action_type == "element_update":
        element_id = action_payload.get("id")
        for i, el in enumerate(session_data.canvas_elements):
            if el.id == element_id:
                session_data.canvas_elements[i] = el.model_copy(update=action_payload)
                break
    elif action_type == "element_remove":
        ids_to_remove = set(action_payload.get("ids", [action_payload.get("id")]))
        session_data.canvas_elements = [el for el in session_data.canvas_elements if el.id not in ids_to_remove]

    # Save the modified object back to the database
    session_data.last_modified = datetime.now(UTC).timestamp()
    canvas_tool.db_mod.set(session_db_key, session_data.model_dump_json(exclude_none=True))

    # Broadcast the successful, persisted action to all connected clients
    # MODIFICATION: Call the non-blocking broadcast method.
    canvas_tool._broadcast_to_canvas_listeners(
        canvas_id=canvas_id,
        event_type="canvas_elements_changed",
        data={"action": action_type, "element": action_payload},
        originator_user_id=user_id
    )

    # Clear the temporary preview of the user who made the change
    async with canvas_tool.previews_lock:
        if user_id in canvas_tool.active_user_previews.get(canvas_id, {}):
            del canvas_tool.active_user_previews[canvas_id][user_id]

    # MODIFICATION: Call the non-blocking broadcast method.
    canvas_tool._broadcast_to_canvas_listeners(
        canvas_id=canvas_id, event_type="clear_user_preview",
        data={"user_id": user_id}, originator_user_id=user_id
    )

    return Result.ok(info=f"Action '{action_type}' persisted and broadcast.")

markdown_to_svg(self, request, markdown_text='', width=400, font_family='sans-serif', font_size=14, bg_color='#ffffff', text_color='#000000') async

Converts a string of Markdown text into an SVG image. The SVG is returned as a base64 encoded data URL. This version uses a viewBox for better scalability and multi-line handling.

Source code in toolboxv2/mods/Canvas.py
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
@export(mod_name=MOD_NAME, api=True, version=VERSION, name="markdown_to_svg", api_methods=['POST'],
        request_as_kwarg=True)
async def markdown_to_svg(self, request: RequestData, markdown_text: str = "", width: int = 400,
                          font_family: str = "sans-serif", font_size: int = 14,
                          bg_color: str = "#ffffff", text_color: str = "#000000") -> Result:
    """
    Converts a string of Markdown text into an SVG image.
    The SVG is returned as a base64 encoded data URL.
    This version uses a viewBox for better scalability and multi-line handling.
    """
    if request is None:
        return Result.default_user_error("Request data is missing.", 400)
    if not markdown_text and request.data:
        markdown_text = request.data.get("markdown_text", "")

    if not markdown_text:
        return Result.default_user_error("markdown_text cannot be empty.")

    try:
        # Convert Markdown to HTML
        html_content = markdown2.markdown(markdown_text, extras=["fenced-code-blocks", "tables", "strike"])

        # --- FIX for Multi-line text ---
        # The key is to NOT set a fixed height on the SVG itself, but to use a viewBox.
        # The client will determine the final rendered size.
        # The width of the div inside the foreignObject controls the line wrapping.

        # We still need a rough height for the viewBox.
        # Estimate height: (number of lines * line-height) + padding
        # A simple line-height estimate is font_size * 1.6
        line_height_estimate = font_size * 1.6
        num_lines_estimate = len(html_content.split('\n')) + html_content.count('<br') + html_content.count(
            '<p>') + html_content.count('<li>')
        estimated_height = (num_lines_estimate * line_height_estimate) + 40  # 20px top/bottom padding

        svg_template = f"""
        <svg viewBox="0 0 {width} {int(estimated_height)}" xmlns="http://www.w3.org/2000/svg">
            <foreignObject x="0" y="0" width="{width}" height="{int(estimated_height)}">
                <div xmlns="http://www.w3.org/1999/xhtml">
                    <style>
                        div {{
                            font-family: {font_family};
                            font-size: {font_size}px;
                            color: {text_color};
                            background-color: {bg_color};
                            padding: 10px;
                            border-radius: 5px;
                            line-height: 1.6;
                            width: {width - 20}px; /* Width minus padding */
                            word-wrap: break-word;
                            height: 100%;
                            overflow-y: auto; /* Allow scrolling if content overflows estimate */
                        }}
                        h1, h2, h3 {{ border-bottom: 1px solid #ccc; padding-bottom: 5px; margin-top: 1em; }}
                        pre {{ background-color: #f0f0f0; padding: 10px; border-radius: 4px; overflow-x: auto; }}
                        code {{ font-family: monospace; }}
                        table {{ border-collapse: collapse; width: 100%; }}
                        th, td {{ border: 1px solid #ddd; padding: 8px; }}
                        th {{ background-color: #f2f2f2; }}
                        blockquote {{ border-left: 4px solid #ccc; padding-left: 10px; color: #555; margin-left: 0; }}
                    </style>
                    {html_content}
                </div>
            </foreignObject>
        </svg>
        """

        svg_base64 = base64.b64encode(svg_template.encode('utf-8')).decode('utf-8')
        data_url = f"data:image/svg+xml;base64,{svg_base64}"

        # --- FIX for Editability ---
        # Return the original markdown text along with the SVG
        return Result.ok(data={"svg_data_url": data_url, "original_markdown": markdown_text})

    except Exception as e:
        self.app.logger.error(f"Error converting Markdown to SVG: {e}", exc_info=True)
        return Result.default_internal_error("Failed to convert Markdown to SVG.")

save_session(app, request, data) async

Saves the entire state of a canvas session to the database. This is typically triggered by a user's explicit "Save" action.

Source code in toolboxv2/mods/Canvas.py
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
@export(mod_name=MOD_NAME, api=True, version=VERSION, name="save_session", api_methods=['POST'], request_as_kwarg=True)
async def save_session(app: App, request: RequestData, data: dict[str, Any] | IdeaSessionData) -> Result:
    """
    Saves the entire state of a canvas session to the database.
    This is typically triggered by a user's explicit "Save" action.
    """
    if not data:
        return Result.default_user_error("Request data is missing.", 400)
    if request is None:
        return Result.default_user_error("Request data is missing.", 400)
    canvas_tool = app.get_mod(MOD_NAME)
    if not canvas_tool or not canvas_tool.db_mod:
        app.logger.error("Save failed: Canvas module or DB not available.")
        return Result.custom_error(info="Database module not available.", exec_code=503)

    user_db_key_base = await canvas_tool._get_user_specific_db_key(request, SESSION_DATA_PREFIX)
    if not user_db_key_base:
        return Result.default_user_error(info="User authentication required to save.", exec_code=401)

    try:
        # Validate the incoming data against the Pydantic model
        session_data_obj = IdeaSessionData(**data) if isinstance(data, dict) else data
    except Exception as e:
        app.logger.error(f"Invalid session data for save: {e}. Data: {str(data)[:500]}", exc_info=True)
        return Result.default_user_error(info=f"Invalid session data format: {e}", exec_code=400)

    # Update timestamp and construct the main session key
    if session_data_obj:
        session_data_obj.last_modified = datetime.now(UTC).timestamp()
    session_db_key = f"{user_db_key_base}_{session_data_obj.id}"

    # Save the full session object to the database
    canvas_tool.db_mod.set(session_db_key, session_data_obj.model_dump_json(exclude_none=True))
    app.logger.info(f"Saved session data for C:{session_data_obj.id}")

    # --- Update the session list metadata ---
    session_list_key = f"{user_db_key_base}{SESSION_LIST_KEY_SUFFIX}"
    try:
        list_res_obj = canvas_tool.db_mod.get(session_list_key)
        user_sessions = []
        if list_res_obj and not list_res_obj.is_error() and list_res_obj.get():
            list_content = list_res_obj.get()[0] if isinstance(list_res_obj.get(), list) else list_res_obj.get()
            user_sessions = json.loads(list_content)

        # Find and update the existing entry, or add a new one
        session_metadata = {
            "id": session_data_obj.id,
            "name": session_data_obj.name,
            "last_modified": session_data_obj.last_modified
        }
        found_in_list = False
        for i, sess_meta in enumerate(user_sessions):
            if sess_meta.get("id") == session_data_obj.id:
                user_sessions[i] = session_metadata
                found_in_list = True
                break
        if not found_in_list:
            user_sessions.append(session_metadata)

        canvas_tool.db_mod.set(session_list_key, json.dumps(user_sessions))
        app.logger.info(f"Updated session list for user key ending in ...{user_db_key_base[-12:]}")

    except Exception as e:
        app.logger.error(f"Failed to update session list for C:{session_data_obj.id}. Error: {e}", exc_info=True)
        # Non-fatal error; the main data was saved. We can continue.

    return Result.ok(
        info="Session saved successfully.",
        data={"id": session_data_obj.id, "last_modified": session_data_obj.last_modified}
    )

ChatModule

get_chat_ui(app)

Liefert das Haupt-HTML-UI für das Chat-Widget. Es verwendet app.web_context(), um das notwendige tbjs CSS und JS einzubinden.

Source code in toolboxv2/mods/ChatModule.py
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
@export(mod_name=Name, version=version, api=True, name="ui", row=True)
def get_chat_ui(app: App) -> Result:
    """
    Liefert das Haupt-HTML-UI für das Chat-Widget.
    Es verwendet `app.web_context()`, um das notwendige tbjs CSS und JS einzubinden.
    """

    html_content = f"""
        {app.web_context()}
        <style>
            body {{
                display: flex;
                align-items: center;
                justify-content: center;
                min-height: 100vh;
                padding: 1rem;
                background-color: var(--theme-bg);
            }}
        </style>
        <main id="chat-container" style="width: 100%; height: 80vh;">
            <!-- Das Chat-Widget wird hier initialisiert -->
        </main>

        <script unsave="true">
            // Verwende TB.once, um sicherzustellen, dass das Framework vollständig initialisiert ist,
            // bevor unser Code ausgeführt wird.
            TB.once(() => {{
                const chatContainer = document.getElementById('chat-container');
                if (chatContainer && TB.ui.ChatWidget) {{
                    // Initialisiere das Chat-Widget in unserem Container
                    TB.ui.ChatWidget.init(chatContainer);

                    // Verbinde mit dem in diesem Modul definierten WebSocket-Endpunkt
                    TB.ui.ChatWidget.connect();
                }} else {{
                    console.error("Chat UI initialization failed: container or ChatWidget not found.");
                }}
            }});
        </script>
    """

    return Result.html(data=html_content)

on_chat_message(app, conn_id, session, payload) async

Wird aufgerufen, wenn eine Nachricht von einem Client empfangen wird.

Source code in toolboxv2/mods/ChatModule.py
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
async def on_chat_message(app: App, conn_id: str, session: dict, payload: dict):
    """
    Wird aufgerufen, wenn eine Nachricht von einem Client empfangen wird.
    """
    username = session.get("user_name", "Anonymous")
    print(f"WS MESSAGE from {username} ({conn_id}): {session}")
    message_text = payload.get("data", {}).get("message", "").strip()

    if not message_text:
        return  # Ignoriere leere Nachrichten

    app.print(f"WS MESSAGE from {username} ({conn_id}): {message_text}")

    # Sende die Nachricht an alle im Raum (einschließlich des Absenders)
    await app.ws_broadcast(
        channel_id="ChatModule/public_room",
        payload={"event": "new_message", "data": {"user": username, "text": message_text}}
    )

on_user_connect(app, conn_id, session) async

Wird vom Rust WebSocket Actor aufgerufen, wenn ein neuer Client eine Verbindung herstellt.

Source code in toolboxv2/mods/ChatModule.py
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
async def on_user_connect(app: App, conn_id: str, session: dict):
    """
    Wird vom Rust WebSocket Actor aufgerufen, wenn ein neuer Client eine Verbindung herstellt.
    """
    username = session.get("user_name", "Anonymous")
    app.print(f"WS CONNECT: User '{username}' connected with conn_id: {conn_id}")

    # Sende eine Willkommensnachricht direkt an den neuen Benutzer (1-zu-1)
    await app.ws_send(conn_id, {"event": "welcome", "data": f"Welcome to the public chat, {username}!"})

    # Kündige den neuen Benutzer allen anderen im Raum an (1-zu-n)
    await app.ws_broadcast(
        channel_id="ChatModule/public_room",
        payload={"event": "user_joined", "data": f"👋 {username} has joined the chat."},
        source_conn_id=conn_id  # Schließt den Absender von diesem Broadcast aus
    )

on_user_disconnect(app, conn_id, session=None) async

Wird aufgerufen, wenn die Verbindung eines Clients geschlossen wird.

Source code in toolboxv2/mods/ChatModule.py
59
60
61
62
63
64
65
66
67
68
69
70
71
72
async def on_user_disconnect(app: App, conn_id: str, session: dict=None):
    """
    Wird aufgerufen, wenn die Verbindung eines Clients geschlossen wird.
    """
    if session is None:
        session = {}
    username = session.get("user_name", "Anonymous")
    app.print(f"WS DISCONNECT: User '{username}' disconnected (conn_id: {conn_id})")

    # Kündige den Weggang des Benutzers allen verbleibenden Benutzern im Raum an
    await app.ws_broadcast(
        channel_id="ChatModule/public_room",
        payload={"event": "user_left", "data": f"😥 {username} has left the chat."}
    )

register_chat_handlers(app)

Registriert die asynchronen Funktionen als Handler für spezifische WebSocket-Ereignisse. Der Funktionsname (register_chat_handlers) ist beliebig. Der Decorator ist entscheidend.

Returns:

Type Description
dict

Ein Dictionary, das Ereignisnamen auf ihre Handler-Funktionen abbildet.

Source code in toolboxv2/mods/ChatModule.py
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
@export(mod_name=Name, version=version, websocket_handler="public_room")
def register_chat_handlers(app: App) -> dict:
    """
    Registriert die asynchronen Funktionen als Handler für spezifische WebSocket-Ereignisse.
    Der Funktionsname (`register_chat_handlers`) ist beliebig. Der Decorator ist entscheidend.

    Returns:
        Ein Dictionary, das Ereignisnamen auf ihre Handler-Funktionen abbildet.
    """
    return {
        "on_connect": on_user_connect,
        "on_message": on_chat_message,
        "on_disconnect": on_user_disconnect,
    }

CloudM

check_multiple_processes(pids)

Checks the status of multiple processes in a single system call. Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).

Source code in toolboxv2/mods/CloudM/mini.py
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def check_multiple_processes(pids: list[int]) -> dict[int, str]:
    """
    Checks the status of multiple processes in a single system call.
    Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).
    """
    if not pids:
        return {}

    pid_status = {}

    if os.name == 'nt':  # Windows
        try:
            # Windows tasklist requires separate /FI for each filter
            command = 'tasklist'

            # Add encoding handling for Windows
            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='cp850'  # Use cp850 for Windows console output
            )
            # Create a set of running PIDs from the output
            running_pids = set()
            for line in result.stdout.lower().split('\n'):
                for pid in pids:
                    if str(pid) in line:
                        running_pids.add(pid)
            # Assign status based on whether PID was found in output
            for pid in pids:
                if pid in running_pids:
                    pid_status[pid] = GREEN_CIRCLE
                else:
                    pid_status[pid] = RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            # Mark all as YELLOW_CIRCLE if there's an error running the command
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE
        except UnicodeDecodeError as e:
            print(f"UnicodeDecodeError: {e}")  # For debugging
            # Try alternate encoding if cp850 fails
            try:
                result = subprocess.run(
                    command,
                    capture_output=True,
                    text=True,
                    shell=True,
                    encoding='utf-8'
                )
                running_pids = set()
                for line in result.stdout.lower().split('\n'):
                    for pid in pids:
                        if str(pid) in line:
                            running_pids.add(pid)

                for pid in pids:
                    pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE
            except Exception as e:
                print(f"Failed with alternate encoding: {e}")  # For debugging
                for pid in pids:
                    pid_status[pid] = YELLOW_CIRCLE

    else:  # Unix/Linux/Mac
        try:
            pids_str = ','.join(str(pid) for pid in pids)
            command = f'ps -p {pids_str} -o pid='

            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='utf-8'
            )
            running_pids = set(int(pid) for pid in result.stdout.strip().split())

            for pid in pids:
                pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE

    return pid_status

get_service_pids(info_dir)

Extracts service names and PIDs from pid files.

Source code in toolboxv2/mods/CloudM/mini.py
14
15
16
17
18
19
20
21
22
23
24
25
26
27
def get_service_pids(info_dir):
    """Extracts service names and PIDs from pid files."""
    services = {}
    pid_files = [f for f in os.listdir(info_dir) if re.match(r'(.+)-(.+)\.pid', f)]
    for pid_file in pid_files:
        match = re.match(r'(.+)-(.+)\.pid', pid_file)
        if match:
            services_type, service_name = match.groups()
            # Read the PID from the file
            with open(os.path.join(info_dir, pid_file)) as file:
                pid = file.read().strip()
                # Store the PID using a formatted key
                services[f"{service_name} - {services_type}"] = int(pid)
    return services

get_service_status(dir)

Displays the status of all services.

Source code in toolboxv2/mods/CloudM/mini.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def get_service_status(dir: str) -> str:
    """Displays the status of all services."""
    if time.time()-services_data_sto_last_update_time[0] > 30:
        services = get_service_pids(dir)
        services_data_sto[0] = services
        services_data_sto_last_update_time[0] = time.time()
    else:
        services = services_data_sto[0]
    if not services:
        return "No services found"

    # Get status for all PIDs in a single call
    pid_statuses = check_multiple_processes(list(services.values()))

    # Build the status string
    res_s = "Service(s):" + ("\n" if len(services) > 1 else ' ')
    for service_name, pid in services.items():
        status = pid_statuses.get(pid, YELLOW_CIRCLE)
        res_s += f"{status} {service_name} (PID: {pid})\n"
    services_data_display[0] = res_s.strip()
    return res_s.rstrip()

AuthManager

delete_user(app, username)

Deletes a user and all their data.

Source code in toolboxv2/mods/CloudM/AuthManager.py
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
@export(mod_name=Name, state=True, test=False, interface=ToolBoxInterfaces.native)
def delete_user(app: App, username: str):
    """Deletes a user and all their data."""
    if not db_helper_test_exist(app, username):
        return Result.default_user_error(f"User '{username}' not found.")

    # This will delete all entries matching the user
    result = db_helper_delete_user(app, username, '*', matching=True)

    if result.is_ok():
        # Also remove the local private key file if it exists
        app.config_fh.remove_key_file_handler("Pk" + Code.one_way_hash(username, "dvp-k")[:8])
        return Result.ok(f"User '{username}' deleted successfully.")
    else:
        return Result.default_internal_error(f"Failed to delete user '{username}'.", data=result)
list_users(app)

Lists all registered users.

Source code in toolboxv2/mods/CloudM/AuthManager.py
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
@export(mod_name=Name, state=True, test=False, interface=ToolBoxInterfaces.native)
def list_users(app: App):
    """Lists all registered users."""
    keys_result = app.run_any(TBEF.DB.GET, query="all-k", get_results=True)
    if keys_result.is_error():
        return keys_result

    user_keys = keys_result.get()
    if not user_keys:
        return Result.ok("No users found.")

    users = []
    for key in user_keys:
        if isinstance(key, bytes):
            key = key.decode()
        if not key.startswith("USER::"):
            continue
        # Extract username from the key USER::username::uid
        parts = key.split('::')
        if len(parts) > 1 and parts[1] not in [u['username'] for u in users]:
            user_res = get_user_by_name(app, parts[1])
            if user_res.is_ok():
                user_data = user_res.get()
                users.append({"username": user_data.name, "email": user_data.email, "level": user_data.level})

    return Result.ok(data=users)

LogInSystem

cli_logout(app=None) async

Enhanced logout with modern visual feedback

Source code in toolboxv2/mods/CloudM/LogInSystem.py
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
async def cli_logout(app: App = None):
    """Enhanced logout with modern visual feedback"""
    if app is None:
        app = get_app("CloudM.cli_logout")

    # Clear screen
    print('\033[2J\033[H')

    print_box_header("Logout Process", "🔓")
    print_box_content("Terminating session...", "info")
    print_box_footer()

    username = app.get_username()

    if not username:
        print_status("No active session found", "warning")
        return Result.ok("No session to logout")

    print_status(f"Logging out user: {username}", "progress")
    with Spinner("Finalizing logout..."):
        # Close CLI session
        from .UserInstances import UserInstances, close_cli_session

        try:
            cli_session_id = UserInstances.get_cli_session_id(username).get()
            close_cli_session(cli_session_id)
            print_status("CLI session closed", "success")
        except Exception as e:
            print_status(f"Session cleanup warning: {e}", "warning")

        # Clear JWT token
        from toolboxv2.utils.extras.blobs import BlobFile
        try:
            with BlobFile(f"claim/{username}/jwt.c", key=Code.DK()(), mode="w") as blob:
                blob.clear()
            print_status("Credentials cleared", "success")
        except Exception as e:
            print_status(f"Credential cleanup warning: {e}", "warning")

        # Clear session
        if app.session:
            app.session.valid = False
            app.session.username = None

    print()
    print_box_header("Logout Complete", "✓")
    print_box_content(f"User '{username}' logged out successfully", "success")
    print_box_content("All credentials have been cleared", "info")
    print_box_footer()

    return Result.ok("Logout successful")
cli_web_login(app=None, force_remote=False, force_local=False) async

Enhanced CLI web login with remote/local options and modern visual feedback

Source code in toolboxv2/mods/CloudM/LogInSystem.py
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
async def cli_web_login(app: App = None, force_remote: bool = False, force_local: bool = False):
    """
    Enhanced CLI web login with remote/local options and modern visual feedback
    """
    if app is None:
        app = get_app("CloudM.cli_web_login")

    # Check if already logged in
    if app.session and app.session.valid:
        print_box_header("Already Authenticated", "✓")
        print_box_content("You are already logged in!", "success")
        print_box_content(f"Username: {app.get_username()}", "info")
        print_box_footer()
        return Result.ok("Already authenticated")

    # Determine login method
    login_method = await _determine_login_method(app, force_remote, force_local)

    if login_method == "remote":
        return await _remote_web_login(app)
    elif login_method == "local":
        return await _local_web_login(app)
    else:
        print_status("Login cancelled by user", "warning")
        return Result.default_user_error("Login cancelled by user")
open_check_cli_auth(session_id, app=None) async

Check if CLI authentication is complete

Source code in toolboxv2/mods/CloudM/LogInSystem.py
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
@export(mod_name=Name, version=version, api=True)
async def open_check_cli_auth(session_id: str, app: App = None):
    """Check if CLI authentication is complete"""
    if app is None:
        app = get_app("CloudM.open_check_cli_auth")

    # Check session storage for completed authentication
    auth_data_str = await _get_db_helper(app, f"cli_auth_{session_id}")

    if auth_data_str:
        try:
            auth_data = json.loads(auth_data_str)
            return {
                'authenticated': True,
                'jwt_token': auth_data.get('jwt_token'),
                'username': auth_data.get('username')
            }
        except (json.JSONDecodeError, TypeError) as e:
            app.logger.error(f"Error parsing CLI auth data for session {session_id}: {e}")
            return {'authenticated': False}

    return {'authenticated': False}
open_complete_cli_auth(session_id, jwt_token, username, app=None) async

Complete CLI authentication process - No auth required as this IS the auth process

Source code in toolboxv2/mods/CloudM/LogInSystem.py
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
@export(mod_name=Name, version=version, api=True)
async def open_complete_cli_auth(session_id: str, jwt_token: str, username: str, app: App = None):
    """Complete CLI authentication process - No auth required as this IS the auth process"""
    if app is None:
        app = get_app("CloudM.open_complete_cli_auth")

    app.logger.info(f"CLI auth completion requested for session {session_id}, user {username}")

    # Store authentication data temporarily
    auth_data = {
        'jwt_token': jwt_token,
        'username': username,
        'timestamp': time.time()
    }

    try:
        await _set_db_helper(app, {"query": f"cli_auth_{session_id}", "value": json.dumps(auth_data)})
        app.logger.info(f"CLI auth data stored for session {session_id}")

        # Clean up after 10 minutes
        asyncio.create_task(_cleanup_auth_session(app, session_id, 600))

        return Result.ok("CLI authentication completed")
    except Exception as e:
        app.logger.error(f"Error storing CLI auth data for session {session_id}: {e}")
        return Result.default_internal_error(f"Failed to complete CLI authentication: {str(e)}")
open_web_login_web(app, request, session_id=None, return_to=None, log_in_for='CLI') async

Handle web login flow for CLI and Browser

Source code in toolboxv2/mods/CloudM/LogInSystem.py
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
@export(mod_name=Name, version=version, api=True, request_as_kwarg=True)
async def open_web_login_web(app: App, request: RequestData, session_id=None, return_to=None, log_in_for="CLI"):
    """Handle web login flow for CLI and Browser"""
    if request is None:
        return Result.default_internal_error("No request specified")

    template = """<div">
        <div class="login-card">
            <h1>🔐 XXX Authentication</h1>
            <p>Please authenticate to continue with XXX access</p>

            <div id="loginForm">
                <input type="text" id="username" placeholder="Username" required>
                <button type="button" onclick="startLogin()">Login</button>
            </div>

            <div id="statusMessage" style="display: none; padding: 10px; margin: 10px 0; border-radius: 5px;"></div>
            <div id="successMessage" style="display: none;">
                <h3>✅ Authentication Successful!</h3>
                <p>You can now return to your XXX. This window will close automatically.</p>
            </div>
        </div>

    <script unsave="true">

        const urlParams = new URLSearchParams(window.location.search);
        const sessionId = urlParams.get('session_id');
        const returnTo = urlParams.get('return_to');

        console.log('[XXX Login] Session ID:', sessionId);
        console.log('[XXX Login] Return to:', returnTo);

        async function startLogin() {
            const username = document.getElementById('username').value;
            if (!username) {
                showStatus('Please enter a username', 'error');
                return;
            }

            if (!sessionId) {
                showStatus('Error: No session ID provided. Please restart the XXX login process.', 'error');
                return;
            }

            showStatus('Authenticating...', 'info');

            try {
                // Use TB.js login system
                console.log('[XXX Login] Attempting login for:', username);
                const result = await window.TB.user.loginWithDeviceKey(username);
                console.log('[XXX Login] Login result:', result);

                if (result.success) {
                    showStatus('Login successful! Notifying XXX...', 'info');

                    // Get the JWT token from TB.user state
                    const jwtToken = window.TB.user.getToken();
                    console.log('[XXX Login] JWT Token:', jwtToken ? 'Found' : 'Not found');

                    if (!jwtToken) {
                        showStatus('Error: No authentication token found. Please try again.', 'error');
                        return;
                    }

                    // Complete XXX authentication by notifying the backend
                    console.log('[XXX Login] Calling complete_cli_auth with session_id:', sessionId);
                    const completeResponse = await window.TB.api.request(
                        'CloudM',
                        'open_complete_cli_auth',
                        {
                            session_id: sessionId,
                            jwt_token: jwtToken,
                            username: username
                        },
                        'POST'
                    );

                    console.log('[XXX Login] complete_cli_auth response:', completeResponse);

                    if (completeResponse.error === window.TB.ToolBoxError.none) {
                        showSuccess();
                        console.log('[XXX Login] XXX authentication completed successfully');
                        setTimeout(() => {
                            console.log('[XXX Login] Closing window...');
                            window.close();
                        }, 3000);
                    } else {
                        showStatus('Error notifying XXX: ' + (completeResponse.info?.help_text || 'Unknown error'), 'error');
                    }
                } else {
                    showStatus(result.message || 'Login failed', 'error');
                }
            } catch (error) {
                console.error('[XXX Login] Error:', error);
                showStatus('Authentication error: ' + error.message, 'error');
            }
        }

        function showStatus(message, type) {
            const statusEl = document.getElementById('statusMessage');
            statusEl.textContent = message;
            statusEl.style.display = 'block';

            // Style based on type
            if (type === 'error') {
                statusEl.style.backgroundColor = '#fee';
                statusEl.style.color = '#c00';
                statusEl.style.border = '1px solid #c00';
            } else if (type === 'info') {
                statusEl.style.backgroundColor = '#eef';
                statusEl.style.color = '#006';
                statusEl.style.border = '1px solid #006';
            } else if (type === 'success') {
                statusEl.style.backgroundColor = '#efe';
                statusEl.style.color = '#060';
                statusEl.style.border = '1px solid #060';
            }
        }

        function showSuccess() {
            document.getElementById('loginForm').style.display = 'none';
            document.getElementById('statusMessage').style.display = 'none';
            document.getElementById('successMessage').style.display = 'block';
        }

        // Auto-focus username field
        if (document.getElementById('username')) {
            document.getElementById('username').focus();

            // Allow Enter key to submit
            document.getElementById('username').addEventListener('keypress', function(e) {
                if (e.key === 'Enter') {
                e.preventDefault();
                    startLogin();
                }
            });
        }

        // Check if TB is available
        if (!window.TB) {
            console.error('[XXX Login] TB framework not loaded!');
            showStatus('Error: TB framework not loaded. Please refresh the page.', 'error');
        } else {
            console.log('[XXX Login] TB framework loaded successfully');
        }


    </script>
</div>
""".replace("XXX", log_in_for)

    return Result.html(template)
print_menu_option(number, text, selected=False)

Print a menu option

Source code in toolboxv2/mods/CloudM/LogInSystem.py
28
29
30
31
32
33
def print_menu_option(number: int, text: str, selected: bool = False):
    """Print a menu option"""
    if selected:
        print(f"  ▶ {number}. {text}")
    else:
        print(f"    {number}. {text}")

ModManager

CloudM - Advanced Module Manager Production-ready module management system with multi-platform support Version: 0.1.0

ConfigVersion

Bases: Enum

Configuration file versions

Source code in toolboxv2/mods/CloudM/ModManager.py
53
54
55
56
class ConfigVersion(Enum):
    """Configuration file versions"""
    V1 = "1.0"
    V2 = "2.0"
MenuCategory dataclass

Menu category.

Source code in toolboxv2/mods/CloudM/ModManager.py
1360
1361
1362
1363
1364
1365
@dataclass
class MenuCategory:
    """Menu category."""
    name: str
    icon: str
    items: List[MenuItem]
MenuItem dataclass

Menu item with action.

Source code in toolboxv2/mods/CloudM/ModManager.py
1349
1350
1351
1352
1353
1354
1355
1356
1357
@dataclass
class MenuItem:
    """Menu item with action."""
    key: str
    label: str
    action: Callable
    category: str = ""
    icon: str = "•"
    description: str = ""
ModernMenuManager

Modern menu manager with arrow key navigation.

Source code in toolboxv2/mods/CloudM/ModManager.py
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
class ModernMenuManager:
    """Modern menu manager with arrow key navigation."""

    def __init__(self, app_instance: Optional[Any] = None):
        self.app_instance = app_instance
        self.selected_index = 0
        self.categories: List[MenuCategory] = []
        self.flat_items: List[MenuItem] = []
        self.running = True

    def add_category(self, category: MenuCategory):
        """Add a menu category."""
        self.categories.append(category)
        self.flat_items.extend(category.items)

    def get_menu_text(self) -> List[tuple]:
        """Generate formatted menu text."""
        lines = []

        # Header
        lines.append(('class:menu-border', '╔' + '═' * 68 + '╗\n'))
        lines.append(('class:menu-border', '║'))
        lines.append(('class:menu-title', '  🌩️  CloudM - Module Manager'.center(68)))
        lines.append(('class:menu-border', '║\n'))
        lines.append(('class:menu-border', '╠' + '═' * 68 + '╣\n'))

        # Menu items by category
        current_flat_index = 0

        for cat_idx, category in enumerate(self.categories):
            # Category header
            if cat_idx > 0:
                lines.append(('class:menu-border', '║' + '─' * 68 + '║\n'))

            lines.append(('class:menu-border', '║ '))
            lines.append(('class:menu-category', f'{category.icon} {category.name}'))
            lines.append(('', ' ' * (67 - len(category.name) - len(category.icon)- (2 if len(category.icon) == 1 else 1))))
            lines.append(('class:menu-border', '║\n'))

            # Category items
            for item in category.items:
                is_selected = current_flat_index == self.selected_index

                lines.append(('class:menu-border', '║ '))

                if is_selected:
                    lines.append(('class:menu-item-selected', f' ▶ '))
                else:
                    lines.append(('', '   '))

                # Key
                if is_selected:
                    lines.append(('class:menu-item-selected', f'{item.key:>3}'))
                else:
                    lines.append(('class:menu-key', f'{item.key:>3}'))

                # Label

                if is_selected:
                    lines.append(('class:menu-item-selected', f' {item.icon} {item.label}'))
                    remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                    lines.append(('class:menu-item-selected', ' ' * remaining))
                else:
                    lines.append(('class:menu-item', f' {item.icon} {item.label}'))
                    remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                    lines.append(('', ' ' * remaining))

                lines.append(('class:menu-border', '║\n'))
                current_flat_index += 1

        # Footer
        lines.append(('class:menu-border', '╚' + '═' * 68 + '╝\n'))
        lines.append(('class:footer', '\n  ↑↓ or w/s: Navigate  │  Enter: Select  │  q: Quit\n'))

        return lines

    def move_up(self):
        """Move selection up."""
        if self.selected_index > 0:
            self.selected_index -= 1

    def move_down(self):
        """Move selection down."""
        if self.selected_index < len(self.flat_items) - 1:
            self.selected_index += 1

    def get_selected_item(self) -> Optional[MenuItem]:
        """Get currently selected menu item."""
        if 0 <= self.selected_index < len(self.flat_items):
            return self.flat_items[self.selected_index]
        return None

    async def run(self):
        """Run the menu manager."""
        # Build menu structure
        self._build_menu()

        while self.running:
            # Clear screen
            print('\033[2J\033[H')

            # Display menu
            menu_text = self.get_menu_text()
            print_formatted_text(FormattedText(menu_text), style=MODERN_STYLE)

            # Key bindings
            kb = KeyBindings()

            @kb.add('up')
            @kb.add('w')
            def move_up_handler(event):
                self.move_up()
                event.app.exit()

            @kb.add('down')
            @kb.add('s')
            def move_down_handler(event):
                self.move_down()
                event.app.exit()

            @kb.add('enter')
            def select_handler(event):
                event.app.exit(result='select')

            @kb.add('q')
            @kb.add('escape')
            def quit_handler(event):
                event.app.exit(result='quit')

            # Wait for input
            dummy_app = Application(
                layout=Layout(Window(FormattedTextControl(''))),
                key_bindings=kb,
                full_screen=False
            )

            result = await dummy_app.run_async()

            if result == 'quit':
                if await show_confirm('Exit Manager', 'Are you sure you want to exit?'):
                    self.running = False
                    break
            elif result == 'select':
                selected = self.get_selected_item()
                if selected:
                    try:
                        await selected.action()
                    except KeyboardInterrupt:
                        continue
                    except Exception as e:
                        await show_message('Error', f'An error occurred:\n\n{str(e)}', 'error')

    def _build_menu(self):
        """Build menu structure with all operations."""

        # =================== MODULE OPERATIONS ===================
        module_ops = MenuCategory(
            name="MODULE OPERATIONS",
            icon="📦",
            items=[
                MenuItem("1", "List all modules", self._list_modules, icon="📋"),
                MenuItem("2", "Install/Update module", self._install_module, icon="📥"),
                MenuItem("3", "Uninstall module", self._uninstall_module, icon="🗑️"),
                MenuItem("4", "Build installer", self._build_installer, icon="🔨"),
                MenuItem("5", "Upload module", self._upload_module, icon="☁️"),
                MenuItem("6", "Update ALL modules", self._update_all, icon="🔄"),
                MenuItem("7", "Build ALL modules", self._build_all, icon="🏗️"),
            ]
        )

        # =================== CONFIGURATION ===================
        config_ops = MenuCategory(
            name="CONFIGURATION",
            icon="⚙️",
            items=[
                MenuItem("8", "View module info", self._view_info, icon="ℹ️"),
                MenuItem("9", "Validate config", self._validate_config, icon="✓ "),
                MenuItem("10", "Create new config", self._create_config, icon="✎ "),
                MenuItem("11", "Generate ALL configs", self._generate_all_configs, icon="⚡"),
                MenuItem("12", "Generate config for module", self._generate_single_config, icon="⚙️"),
            ]
        )

        # =================== PLATFORM & TEMPLATES ===================
        platform_ops = MenuCategory(
            name="PLATFORM & TEMPLATES",
            icon="🌐",
            items=[
                MenuItem("13", "Build platform installer", self._build_platform, icon="🖥️"),
                MenuItem("14", "Install for platform", self._install_platform, icon="💾"),
                MenuItem("15", "Create from template", self._create_from_template, icon="🎨"),
                MenuItem("16", "List templates", self._list_templates, icon="📚"),
            ]
        )

        self.add_category(module_ops)
        self.add_category(config_ops)
        self.add_category(platform_ops)

    # =================== Action Handlers ===================

    async def _list_modules(self):
        """List all modules."""
        await show_progress("Loading Modules", "Scanning module directory...")

        mods = self.app_instance.get_all_mods()

        if not mods:
            await show_message("No Modules", "No modules found in the directory.", "warning")
            return

        # Build module list
        lines = [f"\n{'#':<4} {'Status':<8} {'Module Name':<35} {'Version':<10}"]
        lines.append('─' * 75)

        for i, mod in enumerate(mods, 1):
            mod_obj = self.app_instance.get_mod(mod)
            ver = getattr(mod_obj, 'version', '?.?.?') if mod_obj else '?.?.?'

            # Check config
            config_path = Path('./mods') / mod / 'tbConfig.yaml'
            single_config = Path('./mods') / f'{mod}.yaml'
            status = "✓ OK" if (config_path.exists() or single_config.exists()) else "✗ No cfg"

            lines.append(f"{i:<4} {status:<8} {mod:<35} {ver:<10}")

        lines.append('─' * 75)
        lines.append(f"\nTotal: {len(mods)} modules")

        await show_message(f"📦 Available Modules ({len(mods)})", '\n'.join(lines), "info")

    async def _install_module(self):
        """Install or update a module."""
        module_name = await show_input("Install Module", "Enter module name:")

        if not module_name:
            return

        await show_progress("Installing", f"Installing module '{module_name}'...")

        result = await installer(self.app_instance, module_name)

        if result.is_error:
            await show_message("Installation Failed", f"Error: {result}", "error")
        else:
            await show_message("Success", f"Module '{module_name}' installed successfully!", "success")

    async def _uninstall_module(self):
        """Uninstall a module."""
        module_name = await show_input("Uninstall Module", "Enter module name:")

        if not module_name:
            return

        if not await show_confirm("Confirm Uninstall", f"Really uninstall '{module_name}'?"):
            return

        await show_progress("Uninstalling", f"Removing module '{module_name}'...")

        result = uninstaller(self.app_instance, module_name)

        if result.is_error:
            await show_message("Uninstall Failed", f"Error: {result}", "error")
        else:
            await show_message("Success", f"Module '{module_name}' uninstalled successfully!", "success")

    async def _build_installer(self):
        """Build module installer."""
        module_name = await show_input("Build Installer", "Enter module name:")

        if not module_name:
            return

        upload = await show_confirm("Upload", "Upload after building?")

        await show_progress("Building", f"Building installer for '{module_name}'...")

        result = await make_installer(self.app_instance, module_name, upload=upload)

        if result.is_error:
            await show_message("Build Failed", f"Error: {result}", "error")
        else:
            msg = f"Installer built successfully!"
            if upload:
                msg += "\n\nModule uploaded to cloud!"
            await show_message("Success", msg, "success")

    async def _upload_module(self):
        """Upload module to cloud."""
        module_name = await show_input("Upload Module", "Enter module name:")

        if not module_name:
            return

        await show_progress("Uploading", f"Uploading '{module_name}' to cloud...")

        result = await upload(self.app_instance, module_name)

        if result.is_error:
            await show_message("Upload Failed", f"Error: {result}", "error")
        else:
            await show_message("Success", f"Module '{module_name}' uploaded successfully!", "success")

    async def _update_all(self):
        """Update all modules."""
        if not await show_confirm(
            "Batch Update",
            "This will update ALL modules.\nThis may take several minutes.\n\nContinue?"
        ):
            return

        await show_progress("Batch Update", "Updating all modules... Please wait.")

        result = await update_all_mods(self.app_instance)

        if result.is_error:
            await show_message("Update Completed", f"Completed with errors:\n\n{result}", "warning")
        else:
            await show_message("Success", "All modules updated successfully!", "success")

    async def _build_all(self):
        """Build all modules."""
        upload = await show_confirm("Upload", "Upload after building?")

        if not await show_confirm(
            "Batch Build",
            "This will build ALL modules.\nThis may take several minutes.\n\nContinue?"
        ):
            return

        await show_progress("Batch Build", "Building all modules... Please wait.")

        result = await build_all_mods(self.app_instance, upload=upload)

        if result.is_error:
            await show_message("Build Completed", f"Completed with errors:\n\n{result}", "warning")
        else:
            msg = "All modules built successfully!"
            if upload:
                msg += "\n\nAll modules uploaded to cloud!"
            await show_message("Success", msg, "success")

    async def _view_info(self):
        """View module information."""
        module_name = await show_input("Module Info", "Enter module name:")

        if not module_name:
            return

        await show_progress("Loading", f"Fetching info for '{module_name}'...")

        result = await get_mod_info(self.app_instance, module_name)

        if result.is_error:
            await show_message("Error", f"Could not get module info:\n\n{result}", "error")
        else:
            info_text = yaml.dump(result.get(), default_flow_style=False, allow_unicode=True)
            await show_message(f"Module Info: {module_name}", info_text, "info")

    async def _validate_config(self):
        """Validate module configuration."""
        module_name = await show_input("Validate Config", "Enter module name:")

        if not module_name:
            return

        config_path = Path('./mods') / module_name / 'tbConfig.yaml'
        if not config_path.exists():
            config_path = Path('./mods') / f'{module_name}.yaml'

        if not config_path.exists():
            await show_message("Error", f"Config file not found for '{module_name}'", "error")
            return

        await show_progress("Validating", f"Checking configuration...")

        config, errors = load_and_validate_config(config_path)

        if errors:
            error_text = '\n'.join([f"  {i}. {err}" for i, err in enumerate(errors, 1)])
            await show_message("Validation Failed", f"Errors found:\n\n{error_text}", "error")
        else:
            await show_message("Success", f"Configuration is valid! ✓", "success")

    async def _create_config(self):
        """Create new module configuration."""
        module_name = await show_input("Create Config", "Module name:")
        if not module_name:
            return

        version = await show_input("Version", "Version:", "0.0.1")
        description = await show_input("Description", "Description (optional):")
        author = await show_input("Author", "Author (optional):")

        # Module type selection
        module_type_choice = await show_choice(
            "Module Type",
            "Select module type:",
            [
                ("package", "📦 Package (directory with multiple files)"),
                ("single", "📄 Single (single file module)")
            ]
        )

        if not module_type_choice:
            return

        module_type = ModuleType.SINGLE if module_type_choice == "single" else ModuleType.PACKAGE

        # Create config
        if module_type == ModuleType.PACKAGE:
            config = create_tb_config_v2(
                module_name=module_name,
                version=version,
                module_type=module_type,
                description=description,
                author=author
            )
        else:
            file_path = await show_input("File Path", "Enter file path:")
            if not file_path:
                return

            config = create_tb_config_single(
                module_name=module_name,
                version=version,
                file_path=file_path,
                description=description,
                author=author
            )

        # Save config
        default_path = f"./mods/{module_name}/tbConfig.yaml"
        save_path = await show_input("Save Location", "Save to:", default_path)

        if not save_path:
            return

        try:
            Path(save_path).parent.mkdir(parents=True, exist_ok=True)

            with open(save_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True)

            await show_message("Success", f"Configuration saved to:\n{save_path}", "success")
        except Exception as e:
            await show_message("Error", f"Could not save config:\n\n{str(e)}", "error")

    async def _generate_all_configs(self):
        """Generate configs for all modules."""
        root_dir = await show_input("Root Directory", "Enter root directory:", "./mods")

        if not root_dir:
            return

        # Generation mode
        mode_choice = await show_choice(
            "Generation Mode",
            "Select generation mode:",
            [
                ("interactive", "💬 Interactive (ask for each module)"),
                ("auto", "🤖 Auto (skip existing configs)"),
                ("force", "⚡ Force (overwrite all configs)")
            ]
        )

        if not mode_choice:
            return

        backup = await show_confirm("Backup", "Create backups of existing configs?")

        interactive_mode = mode_choice == "interactive"
        overwrite_mode = mode_choice == "force"

        if not await show_confirm(
            "Confirm Generation",
            f"Mode: {mode_choice.title()}\n"
            f"Backup: {'Yes' if backup else 'No'}\n"
            f"Root: {root_dir}\n\n"
            "Start generation?"
        ):
            return

        await show_progress("Generating", "Generating configs for all modules...")

        result = await generate_configs_for_existing_mods(
            app=self.app_instance,
            root_dir=root_dir,
            backup=backup,
            interactive=interactive_mode,
            overwrite=overwrite_mode
        )

        if result.is_error:
            await show_message("Completed", f"Generation completed with errors:\n\n{result}", "warning")
        else:
            await show_message("Success", "Config generation completed successfully!", "success")

    async def _generate_single_config(self):
        """Generate config for specific module."""
        # Get module list
        mods = self.app_instance.get_all_mods()

        if not mods:
            await show_message("No Modules", "No modules found.", "warning")
            return

        # Build choices
        choices = []
        for mod in mods:
            config_path = Path('./mods') / mod / 'tbConfig.yaml'
            single_config = Path('./mods') / f'{mod}.yaml'
            status = "✓" if (config_path.exists() or single_config.exists()) else "✗"
            choices.append((mod, f"[{status}] {mod}"))

        module_name = await show_choice(
            "Select Module",
            "Choose module to generate config for:",
            choices
        )

        if not module_name:
            return

        # Check if config exists
        module_path = Path('./mods') / module_name
        config_exists = False

        if module_path.is_dir():
            config_exists = (module_path / 'tbConfig.yaml').exists()
        else:
            config_exists = (Path('./mods') / f'{module_name}.yaml').exists()

        force = False
        if config_exists:
            if not await show_confirm(
                "Config Exists",
                f"Config already exists for '{module_name}'.\n\nOverwrite?"
            ):
                return
            force = True

        await show_progress("Generating", f"Generating config for '{module_name}'...")

        result = await generate_single_module_config(
            app=self.app_instance,
            module_name=module_name,
            force=force
        )

        if result.is_error:
            await show_message("Error", f"Generation failed:\n\n{result}", "error")
        else:
            await show_message("Success", f"Config generated for '{module_name}'!", "success")

    async def _build_platform(self):
        """Build platform-specific installer."""
        module_name = await show_input("Platform Build", "Enter module name:")

        if not module_name:
            return

        # Platform selection
        platform_choices = [(p, f"{p.value}") for p in Platform]
        platform = await show_choice(
            "Select Platform",
            "Choose target platform:",
            platform_choices
        )

        if not platform:
            return

        upload = await show_confirm("Upload", "Upload after building?")

        await show_progress("Building", f"Building for {platform.value}...")

        result = await make_installer(
            self.app_instance, module_name,
            upload=upload,
            platform=platform
        )

        if result.is_error:
            await show_message("Error", f"Build failed:\n\n{result}", "error")
        else:
            await show_message("Success", f"Platform-specific installer built!", "success")

    async def _install_platform(self):
        """Install for specific platform."""
        module_name = await show_input("Platform Install", "Enter module name:")

        if not module_name:
            return

        # Platform selection
        platform_choices = [(p, f"{p.value}") for p in Platform]
        platform = await show_choice(
            "Select Platform",
            "Choose target platform:",
            platform_choices
        )

        if not platform:
            return

        await show_progress("Installing", f"Installing for {platform.value}...")

        result = await installer(self.app_instance, module_name, platform=platform)

        if result.is_error:
            await show_message("Error", f"Installation failed:\n\n{result}", "error")
        else:
            await show_message("Success", "Module installed successfully!", "success")

    async def _create_from_template(self):
        """Create module from template."""
        # Get templates
        result = await list_module_templates(self.app_instance)
        templates = result.get()['templates']

        # Build choices
        template_choices = [
            (t['name'], f"{t['name']:<25} - {t['description']}")
            for t in templates
        ]

        selected_template = await show_choice(
            "Select Template",
            "Choose module template:",
            template_choices
        )

        if not selected_template:
            return

        # Collect information
        module_name = await show_input("Module Name", "Enter module name:")
        if not module_name:
            return

        description = await show_input("Description", "Description (optional):")
        version = await show_input("Version", "Version:", "0.0.1")
        author = await show_input("Author", "Author (optional):")
        location = await show_input("Location", "Location:", "./mods")

        external = await show_confirm("External", "Create external to toolbox?")
        create_config = await show_confirm("Config", "Create tbConfig.yaml?")

        await show_progress("Creating", f"Creating {selected_template} module '{module_name}'...")

        result = await create_module_from_blueprint(
            app=self.app_instance,
            module_name=module_name,
            module_type=selected_template,
            description=description,
            version=version,
            location=location,
            author=author,
            create_config=create_config,
            external=external
        )

        if result.is_error:
            await show_message("Error", f"Module creation failed:\n\n{result}", "error")
        else:
            await show_message(
                "Success",
                f"Module '{module_name}' created successfully!\n\nLocation: {location}/{module_name}",
                "success"
            )

    async def _list_templates(self):
        """List available templates."""
        result = await list_module_templates(self.app_instance)
        templates = result.get()['templates']

        lines = []
        for t in templates:
            lines.append(f"\n┌─ {t['name']}")
            lines.append(f"│  Description: {t['description']}")
            lines.append(f"│  Type: {t['type']}")
            lines.append(f"│  Requires: {', '.join(t['requires']) if t['requires'] else 'None'}")
            lines.append("└" + "─" * 60)

        await show_message("📚 Available Templates", '\n'.join(lines), "info")
add_category(category)

Add a menu category.

Source code in toolboxv2/mods/CloudM/ModManager.py
1449
1450
1451
1452
def add_category(self, category: MenuCategory):
    """Add a menu category."""
    self.categories.append(category)
    self.flat_items.extend(category.items)
get_menu_text()

Generate formatted menu text.

Source code in toolboxv2/mods/CloudM/ModManager.py
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
def get_menu_text(self) -> List[tuple]:
    """Generate formatted menu text."""
    lines = []

    # Header
    lines.append(('class:menu-border', '╔' + '═' * 68 + '╗\n'))
    lines.append(('class:menu-border', '║'))
    lines.append(('class:menu-title', '  🌩️  CloudM - Module Manager'.center(68)))
    lines.append(('class:menu-border', '║\n'))
    lines.append(('class:menu-border', '╠' + '═' * 68 + '╣\n'))

    # Menu items by category
    current_flat_index = 0

    for cat_idx, category in enumerate(self.categories):
        # Category header
        if cat_idx > 0:
            lines.append(('class:menu-border', '║' + '─' * 68 + '║\n'))

        lines.append(('class:menu-border', '║ '))
        lines.append(('class:menu-category', f'{category.icon} {category.name}'))
        lines.append(('', ' ' * (67 - len(category.name) - len(category.icon)- (2 if len(category.icon) == 1 else 1))))
        lines.append(('class:menu-border', '║\n'))

        # Category items
        for item in category.items:
            is_selected = current_flat_index == self.selected_index

            lines.append(('class:menu-border', '║ '))

            if is_selected:
                lines.append(('class:menu-item-selected', f' ▶ '))
            else:
                lines.append(('', '   '))

            # Key
            if is_selected:
                lines.append(('class:menu-item-selected', f'{item.key:>3}'))
            else:
                lines.append(('class:menu-key', f'{item.key:>3}'))

            # Label

            if is_selected:
                lines.append(('class:menu-item-selected', f' {item.icon} {item.label}'))
                remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                lines.append(('class:menu-item-selected', ' ' * remaining))
            else:
                lines.append(('class:menu-item', f' {item.icon} {item.label}'))
                remaining = 60 - len(item.label) - len(item.icon) - (2 if len(item.icon) == 1 else 1)
                lines.append(('', ' ' * remaining))

            lines.append(('class:menu-border', '║\n'))
            current_flat_index += 1

    # Footer
    lines.append(('class:menu-border', '╚' + '═' * 68 + '╝\n'))
    lines.append(('class:footer', '\n  ↑↓ or w/s: Navigate  │  Enter: Select  │  q: Quit\n'))

    return lines
get_selected_item()

Get currently selected menu item.

Source code in toolboxv2/mods/CloudM/ModManager.py
1525
1526
1527
1528
1529
def get_selected_item(self) -> Optional[MenuItem]:
    """Get currently selected menu item."""
    if 0 <= self.selected_index < len(self.flat_items):
        return self.flat_items[self.selected_index]
    return None
move_down()

Move selection down.

Source code in toolboxv2/mods/CloudM/ModManager.py
1520
1521
1522
1523
def move_down(self):
    """Move selection down."""
    if self.selected_index < len(self.flat_items) - 1:
        self.selected_index += 1
move_up()

Move selection up.

Source code in toolboxv2/mods/CloudM/ModManager.py
1515
1516
1517
1518
def move_up(self):
    """Move selection up."""
    if self.selected_index > 0:
        self.selected_index -= 1
run() async

Run the menu manager.

Source code in toolboxv2/mods/CloudM/ModManager.py
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
async def run(self):
    """Run the menu manager."""
    # Build menu structure
    self._build_menu()

    while self.running:
        # Clear screen
        print('\033[2J\033[H')

        # Display menu
        menu_text = self.get_menu_text()
        print_formatted_text(FormattedText(menu_text), style=MODERN_STYLE)

        # Key bindings
        kb = KeyBindings()

        @kb.add('up')
        @kb.add('w')
        def move_up_handler(event):
            self.move_up()
            event.app.exit()

        @kb.add('down')
        @kb.add('s')
        def move_down_handler(event):
            self.move_down()
            event.app.exit()

        @kb.add('enter')
        def select_handler(event):
            event.app.exit(result='select')

        @kb.add('q')
        @kb.add('escape')
        def quit_handler(event):
            event.app.exit(result='quit')

        # Wait for input
        dummy_app = Application(
            layout=Layout(Window(FormattedTextControl(''))),
            key_bindings=kb,
            full_screen=False
        )

        result = await dummy_app.run_async()

        if result == 'quit':
            if await show_confirm('Exit Manager', 'Are you sure you want to exit?'):
                self.running = False
                break
        elif result == 'select':
            selected = self.get_selected_item()
            if selected:
                try:
                    await selected.action()
                except KeyboardInterrupt:
                    continue
                except Exception as e:
                    await show_message('Error', f'An error occurred:\n\n{str(e)}', 'error')
ModuleType

Bases: Enum

Module types for different installation strategies

Source code in toolboxv2/mods/CloudM/ModManager.py
46
47
48
49
50
class ModuleType(Enum):
    """Module types for different installation strategies"""
    PACKAGE = "package"  # Full module directory
    SINGLE = "single"  # Single file module
    HYBRID = "hybrid"  # Mix of both
Platform

Bases: Enum

Supported platform types for module installation

Source code in toolboxv2/mods/CloudM/ModManager.py
36
37
38
39
40
41
42
43
class Platform(Enum):
    """Supported platform types for module installation"""
    SERVER = "server"
    CLIENT = "client"
    DESKTOP = "desktop"
    MOBILE = "mobile"
    COMMON = "common"  # Files needed on all platforms
    ALL = "all"
build_all_mods(app, base='mods', upload=True) async

Builds installer packages for all modules.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

required
base str

Base directory containing modules

'mods'
upload bool

Whether to upload packages after building

True

Returns:

Type Description
Result

Result with build summary

Source code in toolboxv2/mods/CloudM/ModManager.py
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
@export(mod_name=Name, name="build_all", test=False)
async def build_all_mods(app: Optional[App], base: str = "mods",
                         upload: bool = True) -> Result:
    """
    Builds installer packages for all modules.

    Args:
        app: Application instance
        base: Base directory containing modules
        upload: Whether to upload packages after building

    Returns:
        Result with build summary
    """
    if app is None:
        app = get_app(f"{Name}.build_all")

    all_mods = app.get_all_mods()
    results = {"success": [], "failed": []}

    async def build_pipeline(mod_name: str):
        try:
            result = await make_installer(app, mod_name, os.path.join('.', base), upload)
            if result.is_error:
                results["failed"].append({"module": mod_name, "reason": str(result)})
            else:
                results["success"].append(mod_name)
            return result
        except Exception as e:
            results["failed"].append({"module": mod_name, "reason": str(e)})
            return Result.default_internal_error(str(e))

    # Build all modules
    build_results = [await build_pipeline(mod) for mod in all_mods]

    return Result.ok({
        "summary": {
            "total": len(all_mods),
            "success": len(results["success"]),
            "failed": len(results["failed"])
        },
        "details": results
    })
create_and_pack_module(path, module_name='', version='-.-.-', additional_dirs=None, yaml_data=None, platform_filter=None)

Creates and packs a module into a ZIP file with platform-specific support.

Parameters:

Name Type Description Default
path str

Path to module directory or file

required
module_name str

Name of the module

''
version str

Module version

'-.-.-'
additional_dirs Optional[Dict]

Additional directories to include

None
yaml_data Optional[Dict]

Configuration data override

None
platform_filter Optional[Platform]

Optional platform filter for packaging

None

Returns:

Type Description
Optional[str]

Path to created ZIP file or None on failure

Source code in toolboxv2/mods/CloudM/ModManager.py
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
def create_and_pack_module(path: str, module_name: str = '', version: str = '-.-.-',
                           additional_dirs: Optional[Dict] = None,
                           yaml_data: Optional[Dict] = None,
                           platform_filter: Optional[Platform] = None) -> Optional[str]:
    """
    Creates and packs a module into a ZIP file with platform-specific support.

    Args:
        path: Path to module directory or file
        module_name: Name of the module
        version: Module version
        additional_dirs: Additional directories to include
        yaml_data: Configuration data override
        platform_filter: Optional platform filter for packaging

    Returns:
        Path to created ZIP file or None on failure
    """
    if additional_dirs is None:
        additional_dirs = {}
    if yaml_data is None:
        yaml_data = {}

    os.makedirs("./mods_sto/temp/", exist_ok=True)

    module_path = Path(path) / module_name

    if not module_path.exists():
        module_path = Path(f"{path}/{module_name}.py")

    temp_dir = Path(tempfile.mkdtemp(dir="./mods_sto/temp"))

    platform_suffix = f"_{platform_filter.value}" if platform_filter else ""
    zip_file_name = f"RST${module_name}&{__version__}§{version}{platform_suffix}.zip"
    zip_path = Path(f"./mods_sto/{zip_file_name}")

    if not module_path.exists():
        print(f"Module path does not exist: {module_path}")
        return None

    try:
        if module_path.is_dir():
            # Package module - create v2 config
            config_data = create_tb_config_v2(
                module_name=module_name,
                version=version,
                **yaml_data
            )

            config_path = module_path / "tbConfig.yaml"
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)

            # Generate requirements
            req_path = module_path / "requirements.txt"
            generate_requirements(str(module_path), str(req_path))

            # Copy module directory
            shutil.copytree(module_path, temp_dir / module_path.name, dirs_exist_ok=True)

        else:
            # Single file module - create single config
            config_data = create_tb_config_single(
                module_name=module_name,
                version=version,
                file_path=str(module_path),
                **yaml_data
            )

            # Copy file
            shutil.copy2(module_path, temp_dir)

            # Create config
            config_path = temp_dir / f"{module_name}.yaml"
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config_data, f, default_flow_style=False, allow_unicode=True)

            # Generate requirements
            req_path = temp_dir / "requirements.txt"
            generate_requirements(str(temp_dir), str(req_path))

        # Add additional directories
        for dir_name, dir_paths in additional_dirs.items():
            if isinstance(dir_paths, str):
                dir_paths = [dir_paths]

            for dir_path in dir_paths:
                dir_path = Path(dir_path)
                full_path = temp_dir / dir_name

                if dir_path.is_dir():
                    shutil.copytree(dir_path, full_path, dirs_exist_ok=True)
                elif dir_path.is_file():
                    full_path.mkdir(parents=True, exist_ok=True)
                    shutil.copy2(dir_path, full_path)
                else:
                    print(f"Path is neither directory nor file: {dir_path}")

        # Create ZIP file
        with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
            for root, _dirs, files in os.walk(temp_dir):
                for file in files:
                    file_path = Path(root) / file
                    arcname = file_path.relative_to(temp_dir)
                    zipf.write(file_path, arcname)

        # Cleanup temporary directory
        shutil.rmtree(temp_dir)

        print(f"✓ Successfully created: {zip_path}")
        return str(zip_path)

    except Exception as e:
        print(f"✗ Error creating module package: {str(e)}")
        if temp_dir.exists():
            shutil.rmtree(temp_dir)
        return None
create_module_from_blueprint(app=None, module_name='', module_type='basic', description='', version='0.0.1', location='./mods', author='', create_config=True, external=False) async

Creates a new module from blueprint template.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

None
module_name str

Name of the new module

''
module_type str

Type of module (basic, async_service, workflow, etc.)

'basic'
description str

Module description

''
version str

Initial version

'0.0.1'
location str

Where to create the module

'./mods'
author str

Module author

''
create_config bool

Whether to create tbConfig.yaml

True
external bool

If True, create external to toolbox structure

False

Returns:

Type Description
Result

Result with creation status

Source code in toolboxv2/mods/CloudM/ModManager.py
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
@export(mod_name=Name, name="create_module", test=False)
async def create_module_from_blueprint(
    app: Optional[App] = None,
    module_name: str = "",
    module_type: str = "basic",
    description: str = "",
    version: str = "0.0.1",
    location: str = "./mods",
    author: str = "",
    create_config: bool = True,
    external: bool = False
) -> Result:
    """
    Creates a new module from blueprint template.

    Args:
        app: Application instance
        module_name: Name of the new module
        module_type: Type of module (basic, async_service, workflow, etc.)
        description: Module description
        version: Initial version
        location: Where to create the module
        author: Module author
        create_config: Whether to create tbConfig.yaml
        external: If True, create external to toolbox structure

    Returns:
        Result with creation status
    """
    if app is None:
        app = get_app(f"{Name}.create_module")

    if not module_name:
        return Result.default_user_error("Module name is required")

    if module_type not in MODULE_TEMPLATES:
        return Result.default_user_error(
            f"Invalid module type. Available: {', '.join(MODULE_TEMPLATES.keys())}"
        )

    template = MODULE_TEMPLATES[module_type]

    # Prepare paths
    location_path = Path(location)

    if template["type"] == "package":
        module_path = location_path / module_name
        module_file = module_path / "__init__.py"
    else:
        module_path = location_path
        module_file = module_path / f"{module_name}.py"

    # Check if module already exists
    if module_file.exists():
        return Result.default_user_error(f"Module already exists: {module_file}")

    try:
        # Create directory structure
        if template["type"] == "package":
            module_path.mkdir(parents=True, exist_ok=True)
            print(f"✓ Created package directory: {module_path}")
        else:
            module_path.mkdir(parents=True, exist_ok=True)
            print(f"✓ Using directory: {module_path}")

        # Generate module content
        content = template["content"].format(
            MODULE_NAME=module_name,
            MODULE_NAME_LOWER=module_name.lower(),
            VERSION=version,
            DESCRIPTION=description or template["description"]
        )

        # Write module file
        with open(module_file, 'w', encoding='utf-8') as f:
            f.write(content)
        print(f"✓ Created module file: {module_file}")

        # Create requirements.txt
        req_path = module_path if template["type"] == "package" else module_path
        requirements = []

        if "async" in template["requires"]:
            requirements.append("aiohttp>=3.8.0")

        if requirements:
            req_file = (module_path if template["type"] == "package" else module_path) / "requirements.txt"
            with open(req_file, 'w', encoding='utf-8') as f:
                f.write('\n'.join(requirements))
            print(f"✓ Created requirements.txt")

        # Create tbConfig.yaml
        if create_config and not external:
            if template["type"] == "package":
                config = create_tb_config_v2(
                    module_name=module_name,
                    version=version,
                    module_type=ModuleType.PACKAGE,
                    description=description or template["description"],
                    author=author,
                    metadata={
                        "template": module_type,
                        "created_at": time.strftime("%Y-%m-%d %H:%M:%S")
                    }
                )
                config_path = module_path / "tbConfig.yaml"
            else:
                config = create_tb_config_single(
                    module_name=module_name,
                    version=version,
                    file_path=str(module_file.relative_to(location_path.parent)),
                    description=description or template["description"],
                    author=author,
                    metadata={
                        "template": module_type,
                        "created_at": time.strftime("%Y-%m-%d %H:%M:%S")
                    }
                )
                config_path = module_path / f"{module_name}.yaml"

            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
            print(f"✓ Created config: {config_path}")

        # Create additional files based on type
        if module_type == "api_endpoint":
            # Create example API documentation
            api_doc = module_path / "API.md" if template["type"] == "package" else module_path / f"{module_name}_API.md"
            with open(api_doc, 'w', encoding='utf-8') as f:
                f.write(f"""# {module_name} API Documentation

## Endpoints

### GET /api/{module_name}/get_items
Get list of items with pagination.

**Query Parameters:**
- `limit` (int, optional): Number of items (default: 10)
- `offset` (int, optional): Pagination offset (default: 0)

**Response:**
```json
{{
  "items": [...],
  "total": 100,
  "limit": 10,
  "offset": 0
}}
```

### POST /api/{module_name}/create_item
Create a new item.

**Request Body:**
```json
{{
  "name": "Item name",
  "description": "Item description"
}}
```

### GET /api/{module_name}/health_check
Health check endpoint.
""")
            print(f"✓ Created API documentation")

        elif module_type == "websocket":
            # Create WebSocket client example
            ws_example = module_path / "client_example.html" if template[
                                                                    "type"] == "package" else module_path / f"{module_name}_client.html"
            with open(ws_example, 'w', encoding='utf-8') as f:
                f.write(f"""<!DOCTYPE html>
<html>
<head>
    <title>{module_name} WebSocket Client</title>
    <script src="/static/tbjs/tb.js"></script>
</head>
<body>
    <h1>{module_name} WebSocket Demo</h1>
    <div id="messages"></div>
    <input type="text" id="messageInput" placeholder="Type a message...">
    <button onclick="sendMessage()">Send</button>

    <script>
        // Connect to WebSocket
        TB.ws.connect('/ws/{module_name}/main', {{
            onOpen: () => {{
                console.log('Connected to {module_name}');
            }},
            onMessage: (data) => {{
                console.log('Message:', data);
                displayMessage(data);
            }}
        }});

        // Listen for specific events
        TB.events.on('ws:event:new_message', ({{ data }}) => {{
            displayMessage(data.data);
        }});

        function sendMessage() {{
            const input = document.getElementById('messageInput');
            TB.ws.send({{
                event: 'message',
                data: {{
                    text: input.value,
                    timestamp: new Date().toISOString()
                }}
            }});
            input.value = '';
        }}

        function displayMessage(msg) {{
            const div = document.getElementById('messages');
            div.innerHTML += `<div>${{JSON.stringify(msg)}}</div>`;
        }}
    </script>
</body>
</html>
""")
            print(f"✓ Created WebSocket client example")

        # Create README
        readme_path = module_path / "README.md" if template[
                                                       "type"] == "package" else module_path / f"{module_name}_README.md"
        with open(readme_path, 'w', encoding='utf-8') as f:
            f.write(f"""# {module_name}

{description or template['description']}

## Version
{version}

## Type
{template['description']}

## Installation

```bash
# Install module
python CloudM.py install {module_name}
```

## Usage

```python
from toolboxv2 import get_app

app = get_app("{module_name}.Example")

# Use module functions
# Example code here
```

## Author
{author or 'ToolBoxV2'}

## Created
{time.strftime("%Y-%m-%d %H:%M:%S")}

## Template
{module_type}
""")
        print(f"✓ Created README.md")

        print(f"\n{'=' * 60}")
        print(f"✓ Module '{module_name}' created successfully!")
        print(f"{'=' * 60}")
        print(f"\nLocation: {module_file}")
        print(f"Type: {template['description']}")
        print(f"Version: {version}")

        if not external:
            print(f"\nNext steps:")
            print(f"1. Review and customize the generated code")
            print(f"2. Install dependencies: pip install -r requirements.txt")
            print(
                f"3. Test the module: python -c 'from toolboxv2 import get_app; app = get_app(\"{module_name}.Test\")'")
            print(f"4. Build installer: python CloudM.py build {module_name}")

        return Result.ok(data={
            "module_name": module_name,
            "type": module_type,
            "location": str(module_file),
            "config_created": create_config,
            "files_created": [
                str(module_file),
                str(readme_path)
            ]
        })

    except Exception as e:
        return Result.default_internal_error(f"Failed to create module: {str(e)}")
create_tb_config_single(module_name, version, file_path, description='', author='', specification=None, dependencies=None, platforms=None, metadata=None)

Creates configuration for single-file modules.

Parameters:

Name Type Description Default
module_name str

Name of the module

required
version str

Module version

required
file_path str

Path to the single file

required
description str

Module description

''
author str

Module author

''
specification Optional[Dict]

File specifications (exports, functions, etc.)

None
dependencies Optional[List]

List of dependencies

None
platforms Optional[List[Platform]]

List of supported platforms

None
metadata Optional[Dict]

Additional metadata

None

Returns:

Type Description
Dict

Configuration dictionary for single file module

Source code in toolboxv2/mods/CloudM/ModManager.py
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
def create_tb_config_single(module_name: str, version: str, file_path: str,
                            description: str = "", author: str = "",
                            specification: Optional[Dict] = None,
                            dependencies: Optional[List] = None,
                            platforms: Optional[List[Platform]] = None,
                            metadata: Optional[Dict] = None) -> Dict:
    """
    Creates configuration for single-file modules.

    Args:
        module_name: Name of the module
        version: Module version
        file_path: Path to the single file
        description: Module description
        author: Module author
        specification: File specifications (exports, functions, etc.)
        dependencies: List of dependencies
        platforms: List of supported platforms
        metadata: Additional metadata

    Returns:
        Configuration dictionary for single file module
    """
    if specification is None:
        specification = {
            "exports": [],
            "functions": [],
            "classes": [],
            "requires": []
        }

    if dependencies is None:
        dependencies = []

    if platforms is None:
        platforms = [Platform.ALL.value]
    else:
        platforms = [p.value if isinstance(p, Platform) else p for p in platforms]

    if metadata is None:
        metadata = {}

    return {
        "version": version,
        "config_version": ConfigVersion.V2.value,
        "module_name": module_name,
        "module_type": ModuleType.SINGLE.value,
        "file_path": file_path,
        "description": description,
        "author": author,
        "license": "MIT",
        "specification": specification,
        "dependencies": dependencies,
        "platforms": platforms,
        "metadata": metadata
    }
create_tb_config_v2(module_name, version, module_type=ModuleType.PACKAGE, description='', author='', license='MIT', homepage='', platforms=None, metadata=None)

Creates a v2 tbConfig with platform-specific file management.

Parameters:

Name Type Description Default
module_name str

Name of the module

required
version str

Module version

required
module_type ModuleType

Type of module (package/single/hybrid)

PACKAGE
description str

Module description

''
author str

Module author

''
license str

Module license

'MIT'
homepage str

Module homepage/repository

''
platforms Optional[Dict]

Platform-specific file configurations

None
metadata Optional[Dict]

Additional metadata

None

Returns:

Type Description
Dict

Configuration dictionary

Source code in toolboxv2/mods/CloudM/ModManager.py
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
def create_tb_config_v2(module_name: str, version: str, module_type: ModuleType = ModuleType.PACKAGE,
                        description: str = "", author: str = "", license: str = "MIT",
                        homepage: str = "", platforms: Optional[Dict] = None,
                        metadata: Optional[Dict] = None) -> Dict:
    """
    Creates a v2 tbConfig with platform-specific file management.

    Args:
        module_name: Name of the module
        version: Module version
        module_type: Type of module (package/single/hybrid)
        description: Module description
        author: Module author
        license: Module license
        homepage: Module homepage/repository
        platforms: Platform-specific file configurations
        metadata: Additional metadata

    Returns:
        Configuration dictionary
    """
    if platforms is None:
        platforms = {
            Platform.COMMON.value: {"files": ["*"], "required": True},
            Platform.SERVER.value: {"files": [], "required": False},
            Platform.CLIENT.value: {"files": [], "required": False},
            Platform.DESKTOP.value: {"files": [], "required": False},
            Platform.MOBILE.value: {"files": [], "required": False}
        }

    if metadata is None:
        metadata = {}

    return {
        "version": version,
        "config_version": ConfigVersion.V2.value,
        "module_name": module_name,
        "module_type": module_type.value,
        "description": description,
        "author": author,
        "license": license,
        "homepage": homepage,
        "dependencies_file": f"./mods/{module_name}/requirements.txt",
        "zip": f"RST${module_name}&{__version__}§{version}.zip",
        "platforms": platforms,
        "metadata": metadata
    }
download_files(urls, directory, desc, print_func, filename=None)

Downloads files from URLs with progress indication.

Parameters:

Name Type Description Default
urls List[str]

List of URLs to download

required
directory str

Target directory

required
desc str

Progress bar description

required
print_func callable

Function for printing messages

required
filename Optional[str]

Optional filename (uses basename if None)

None

Returns:

Type Description
str

Path to last downloaded file

Source code in toolboxv2/mods/CloudM/ModManager.py
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
def download_files(urls: List[str], directory: str, desc: str,
                   print_func: callable, filename: Optional[str] = None) -> str:
    """
    Downloads files from URLs with progress indication.

    Args:
        urls: List of URLs to download
        directory: Target directory
        desc: Progress bar description
        print_func: Function for printing messages
        filename: Optional filename (uses basename if None)

    Returns:
        Path to last downloaded file
    """
    for url in tqdm(urls, desc=desc):
        if filename is None:
            filename = os.path.basename(url)
        print_func(f"Downloading {filename}")
        print_func(f"{url} -> {directory}/{filename}")
        os.makedirs(directory, exist_ok=True)
        urllib.request.urlretrieve(url, f"{directory}/{filename}")
    return f"{directory}/{filename}"
download_mod(app, module_name, platform=None) async

Downloads a module ZIP file.

Parameters:

Name Type Description Default
app App

Application instance

required
module_name str

Name of module to download

required
platform Optional[str]

Optional platform filter

None

Returns:

Type Description
Result

Binary result with ZIP file

Source code in toolboxv2/mods/CloudM/ModManager.py
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
@export(mod_name=Name, name="download_mod", api=True, api_methods=['GET'])
async def download_mod(app: App, module_name: str,
                       platform: Optional[str] = None) -> Result:
    """
    Downloads a module ZIP file.

    Args:
        app: Application instance
        module_name: Name of module to download
        platform: Optional platform filter

    Returns:
        Binary result with ZIP file
    """
    try:
        zip_path_str = find_highest_zip_version(module_name)

        if not zip_path_str:
            return Result.default_user_error(
                f"Module '{module_name}' not found",
                exec_code=404
            )

        zip_path = Path(zip_path_str)

        if not zip_path.exists():
            return Result.default_user_error(
                f"Module file not found: {zip_path}",
                exec_code=404
            )

        return Result.binary(
            data=zip_path.read_bytes(),
            content_type="application/zip",
            download_name=zip_path.name
        )

    except Exception as e:
        return Result.default_internal_error(f"Download failed: {str(e)}")
format_status(status, message)

Format status message with icon.

Source code in toolboxv2/mods/CloudM/ModManager.py
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
def format_status(status: str, message: str) -> HTML:
    """Format status message with icon."""
    icons = {
        'success': '✓',
        'error': '✗',
        'warning': '⚠',
        'info': 'ℹ'
    }
    icon = icons.get(status, '•')
    return HTML(f'<{status}>{icon} {message}</{status}>')
generate_configs_for_existing_mods(app=None, root_dir='./mods', backup=True, interactive=True, overwrite=False) async

Generates tbConfig.yaml files for all existing modules in the mods directory.

Supports: - Package modules (directories) -> tbConfig.yaml (v2) - Single file modules (.py files) -> {module_name}.yaml (single)

Parameters:

Name Type Description Default
app Optional[App]

Application instance

None
root_dir str

Root directory containing modules

'./mods'
backup bool

Create backups of existing configs

True
interactive bool

Ask for confirmation before each operation

True
overwrite bool

Overwrite existing configs without asking

False

Returns:

Type Description
Result

Result with generation summary

Source code in toolboxv2/mods/CloudM/ModManager.py
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
@export(mod_name=Name, name="generate_configs", test=False)
async def generate_configs_for_existing_mods(
    app: Optional[App] = None,
    root_dir: str = './mods',
    backup: bool = True,
    interactive: bool = True,
    overwrite: bool = False
) -> Result:
    """
    Generates tbConfig.yaml files for all existing modules in the mods directory.

    Supports:
    - Package modules (directories) -> tbConfig.yaml (v2)
    - Single file modules (.py files) -> {module_name}.yaml (single)

    Args:
        app: Application instance
        root_dir: Root directory containing modules
        backup: Create backups of existing configs
        interactive: Ask for confirmation before each operation
        overwrite: Overwrite existing configs without asking

    Returns:
        Result with generation summary
    """
    if app is None:
        app = get_app(f"{Name}.generate_configs")

    root_path = Path(root_dir)
    if not root_path.exists():
        return Result.default_user_error(f"Directory not found: {root_dir}")

    results = {
        "generated": [],
        "skipped": [],
        "failed": [],
        "backed_up": []
    }

    def create_backup(config_path: Path) -> bool:
        """Creates a backup of existing config file"""
        if not config_path.exists():
            return False

        backup_path = config_path.with_suffix('.yaml.backup')
        counter = 1
        while backup_path.exists():
            backup_path = config_path.with_suffix(f'.yaml.backup{counter}')
            counter += 1

        shutil.copy2(config_path, backup_path)
        results["backed_up"].append(str(backup_path))
        print(f"  📦 Backup created: {backup_path.name}")
        return True

    def read_requirements(module_path: Path) -> List[str]:
        """Reads dependencies from requirements.txt"""
        req_file = module_path / 'requirements.txt' if module_path.is_dir() else module_path.parent / 'requirements.txt'

        if not req_file.exists():
            return []

        try:
            with open(req_file, 'r', encoding='utf-8') as f:
                return [line.strip() for line in f if line.strip() and not line.startswith('#')]
        except Exception as e:
            print(f"  ⚠ Error reading requirements: {e}")
            return []

    def extract_module_info(module_path: Path, module_name: str) -> Dict[str, Any]:
        """Extracts metadata from module by analyzing the code"""
        info = {
            "version": "0.0.1",
            "description": f"Module {module_name}",
            "author": "",
            "exports": [],
            "dependencies": []
        }

        try:
            # Try to load module to get version
            if module_name in app.get_all_mods():
                mod = app.get_mod(module_name)
                if mod:
                    info["version"] = getattr(mod, 'version', '0.0.1')

            # Analyze Python file for exports
            py_file = module_path if module_path.is_file() else module_path / '__init__.py'
            if not py_file.exists() and module_path.is_dir():
                py_file = module_path / f"{module_name}.py"

            if py_file.exists():
                with open(py_file, 'r', encoding='utf-8') as f:
                    content = f.read()

                    # Extract version
                    import re
                    version_match = re.search(r'version\s*=\s*["\']([^"\']+)["\']', content)
                    if version_match:
                        info["version"] = version_match.group(1)

                    # Extract exports
                    export_matches = re.findall(r'@export\([^)]*name=["\']([^"\']+)["\']', content)
                    info["exports"] = export_matches

                    # Extract docstring
                    docstring_match = re.search(r'"""([^"]+)"""', content)
                    if docstring_match:
                        info["description"] = docstring_match.group(1).strip().split('\n')[0]

            # Get dependencies
            info["dependencies"] = read_requirements(module_path)

        except Exception as e:
            print(f"  ⚠ Error extracting info: {e}")

        return info

    def generate_package_config(module_path: Path, module_name: str) -> bool:
        """Generates tbConfig.yaml for package modules"""
        config_path = module_path / "tbConfig.yaml"

        # Check if config exists
        if config_path.exists() and not overwrite:
            if interactive:
                response = input(f"  Config exists for {module_name}. Overwrite? (y/n/b=backup): ").lower()
                if response == 'n':
                    results["skipped"].append(module_name)
                    print(f"  ⏭  Skipped: {module_name}")
                    return False
                elif response == 'b':
                    create_backup(config_path)
            else:
                results["skipped"].append(module_name)
                print(f"  ⏭  Skipped (exists): {module_name}")
                return False
        elif config_path.exists() and backup:
            create_backup(config_path)

        # Extract module information
        info = extract_module_info(module_path, module_name)

        # Determine platform files
        platform_config = {
            Platform.COMMON.value: {
                "files": ["*"],
                "required": True
            },
            Platform.SERVER.value: {
                "files": [],
                "required": False
            },
            Platform.CLIENT.value: {
                "files": [],
                "required": False
            },
            Platform.DESKTOP.value: {
                "files": [],
                "required": False
            },
            Platform.MOBILE.value: {
                "files": [],
                "required": False
            }
        }

        # Create config
        config = create_tb_config_v2(
            module_name=module_name,
            version=info["version"],
            module_type=ModuleType.PACKAGE,
            description=info["description"],
            author=info["author"],
            platforms=platform_config,
            metadata={
                "exports": info["exports"],
                "auto_generated": True,
                "generated_at": time.strftime("%Y-%m-%d %H:%M:%S")
            }
        )

        # Write config
        try:
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)

            # Generate requirements.txt if not exists
            req_path = module_path / "requirements.txt"
            if not req_path.exists():
                generate_requirements(str(module_path), str(req_path))

            results["generated"].append(module_name)
            print(f"  ✓ Generated: {config_path}")
            return True

        except Exception as e:
            results["failed"].append({"module": module_name, "error": str(e)})
            print(f"  ✗ Failed: {module_name} - {e}")
            return False

    def generate_single_config(file_path: Path, module_name: str) -> bool:
        """Generates {module_name}.yaml for single file modules"""
        config_path = file_path.parent / f"{module_name}.yaml"

        # Check if config exists
        if config_path.exists() and not overwrite:
            if interactive:
                response = input(f"  Config exists for {module_name}. Overwrite? (y/n/b=backup): ").lower()
                if response == 'n':
                    results["skipped"].append(module_name)
                    print(f"  ⏭  Skipped: {module_name}")
                    return False
                elif response == 'b':
                    create_backup(config_path)
            else:
                results["skipped"].append(module_name)
                print(f"  ⏭  Skipped (exists): {module_name}")
                return False
        elif config_path.exists() and backup:
            create_backup(config_path)

        # Extract module information
        info = extract_module_info(file_path, module_name)

        # Create single config
        config = create_tb_config_single(
            module_name=module_name,
            version=info["version"],
            file_path=str(file_path.relative_to(root_path.parent)),
            description=info["description"],
            author=info["author"],
            specification={
                "exports": info["exports"],
                "functions": [],
                "classes": [],
                "requires": info["dependencies"]
            },
            dependencies=info["dependencies"],
            platforms=[Platform.ALL.value],
            metadata={
                "auto_generated": True,
                "generated_at": time.strftime("%Y-%m-%d %H:%M:%S")
            }
        )

        # Write config
        try:
            with open(config_path, 'w', encoding='utf-8') as f:
                yaml.dump(config, f, default_flow_style=False, allow_unicode=True, sort_keys=False)

            results["generated"].append(module_name)
            print(f"  ✓ Generated: {config_path}")
            return True

        except Exception as e:
            results["failed"].append({"module": module_name, "error": str(e)})
            print(f"  ✗ Failed: {module_name} - {e}")
            return False

    # Main processing loop
    print(f"\n🔍 Scanning directory: {root_path}")
    print("=" * 60)

    items = sorted(root_path.iterdir())
    total_items = len(items)

    for idx, item in enumerate(items, 1):
        # Skip hidden files/folders and __pycache__
        if item.name.startswith('.') or item.name == '__pycache__':
            continue

        print(f"\n[{idx}/{total_items}] Processing: {item.name}")

        if item.is_dir():
            # Package module
            module_name = item.name
            generate_package_config(item, module_name)

        elif item.is_file() and item.suffix == '.py':
            # Single file module
            module_name = item.stem
            generate_single_config(item, module_name)

    # Summary
    print("\n" + "=" * 60)
    print("📊 Generation Summary:")
    print(f"  ✓ Generated: {len(results['generated'])}")
    print(f"  ⏭  Skipped:   {len(results['skipped'])}")
    print(f"  ✗ Failed:    {len(results['failed'])}")
    print(f"  📦 Backed up: {len(results['backed_up'])}")

    if results['generated']:
        print("\n✓ Generated configs for:")
        for mod in results['generated']:
            print(f"  - {mod}")

    if results['failed']:
        print("\n✗ Failed to generate configs for:")
        for fail in results['failed']:
            print(f"  - {fail['module']}: {fail['error']}")

    return Result.ok({
        "summary": {
            "total_processed": total_items,
            "generated": len(results['generated']),
            "skipped": len(results['skipped']),
            "failed": len(results['failed']),
            "backed_up": len(results['backed_up'])
        },
        "details": results
    })
generate_single_module_config(app=None, module_name='', force=False) async

Generates config for a single specific module.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

None
module_name str

Name of module to generate config for

''
force bool

Force overwrite without asking

False

Returns:

Type Description
Result

Result with generation status

Source code in toolboxv2/mods/CloudM/ModManager.py
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
@export(mod_name=Name, name="generate_single_config", test=False)
async def generate_single_module_config(
    app: Optional[App] = None,
    module_name: str = "",
    force: bool = False
) -> Result:
    """
    Generates config for a single specific module.

    Args:
        app: Application instance
        module_name: Name of module to generate config for
        force: Force overwrite without asking

    Returns:
        Result with generation status
    """
    if app is None:
        app = get_app(f"{Name}.generate_single_config")

    if not module_name:
        return Result.default_user_error("Module name is required")

    # Find module path
    module_path = Path('./mods') / module_name

    if not module_path.exists():
        # Try as single file
        module_path = Path(f'./mods/{module_name}.py')
        if not module_path.exists():
            return Result.default_user_error(f"Module not found: {module_name}")

    print(f"\n🔧 Generating config for: {module_name}")

    # Use the main function with specific parameters
    result = await generate_configs_for_existing_mods(
        app=app,
        root_dir=str(module_path.parent),
        backup=True,
        interactive=not force,
        overwrite=force
    )

    return result
get_mod_info(app, module_name) async

Gets detailed information about a module.

Parameters:

Name Type Description Default
app App

Application instance

required
module_name str

Name of module

required

Returns:

Type Description
Result

Result with module information

Source code in toolboxv2/mods/CloudM/ModManager.py
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
@export(mod_name=Name, name="getModInfo", api=True, api_methods=['GET'])
async def get_mod_info(app: App, module_name: str) -> Result:
    """
    Gets detailed information about a module.

    Args:
        app: Application instance
        module_name: Name of module

    Returns:
        Result with module information
    """
    try:
        zip_path = find_highest_zip_version(module_name)

        if not zip_path:
            return Result.default_user_error(
                f"Module '{module_name}' not found",
                exec_code=404
            )

        # Extract and read config
        with zipfile.ZipFile(zip_path, 'r') as zf:
            config_files = [f for f in zf.namelist() if f.endswith('tbConfig.yaml') or f.endswith('.yaml')]

            if not config_files:
                return Result.default_user_error("No configuration file found in module")

            config_content = zf.read(config_files[0])
            config = yaml.safe_load(config_content)

        return Result.ok(config)

    except Exception as e:
        return Result.default_internal_error(f"Failed to get module info: {str(e)}")
get_mod_version(app, module_name) async

Gets the latest version of a module.

Parameters:

Name Type Description Default
app App

Application instance

required
module_name str

Name of module

required

Returns:

Type Description
Result

Result with version string

Source code in toolboxv2/mods/CloudM/ModManager.py
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
@export(mod_name=Name, name="getModVersion", api=True, api_methods=['GET'])
async def get_mod_version(app: App, module_name: str) -> Result:
    """
    Gets the latest version of a module.

    Args:
        app: Application instance
        module_name: Name of module

    Returns:
        Result with version string
    """
    try:
        version_str = find_highest_zip_version(module_name, version_only=True)

        if version_str:
            return Result.text(version_str)

        return Result.default_user_error(
            f"No build found for module '{module_name}'",
            exec_code=404
        )

    except Exception as e:
        return Result.default_internal_error(f"Failed to get version: {str(e)}")
get_platform_files(config, platform)

Extracts file list for specific platform from config.

Parameters:

Name Type Description Default
config Dict

Module configuration dictionary

required
platform Platform

Target platform

required

Returns:

Type Description
List[str]

List of files for the platform

Source code in toolboxv2/mods/CloudM/ModManager.py
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
def get_platform_files(config: Dict, platform: Platform) -> List[str]:
    """
    Extracts file list for specific platform from config.

    Args:
        config: Module configuration dictionary
        platform: Target platform

    Returns:
        List of files for the platform
    """
    platforms = config.get("platforms", {})

    # Get common files (required on all platforms)
    common_files = platforms.get(Platform.COMMON.value, {}).get("files", [])

    # Get platform-specific files
    platform_files = platforms.get(platform.value, {}).get("files", [])

    return common_files + platform_files
increment_version(version_str, max_value=99)

Increments a version number in the format "vX.Y.Z".

Parameters:

Name Type Description Default
version_str str

Current version number (e.g., "v0.0.1")

required
max_value int

Maximum number per position (default: 99)

99

Returns:

Type Description
str

Incremented version number

Raises:

Type Description
ValueError

If version format is invalid

Source code in toolboxv2/mods/CloudM/ModManager.py
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
def increment_version(version_str: str, max_value: int = 99) -> str:
    """
    Increments a version number in the format "vX.Y.Z".

    Args:
        version_str: Current version number (e.g., "v0.0.1")
        max_value: Maximum number per position (default: 99)

    Returns:
        Incremented version number

    Raises:
        ValueError: If version format is invalid
    """
    if not version_str.startswith("v"):
        raise ValueError("Version must start with 'v' (e.g., 'v0.0.1')")

    version_core = version_str[1:]
    try:
        parsed_version = Version(version_core)
    except ValueError as e:
        raise ValueError(f"Invalid version number: {version_core}") from e

    parts = list(parsed_version.release)

    # Increment rightmost position
    for i in range(len(parts) - 1, -1, -1):
        if parts[i] < max_value:
            parts[i] += 1
            break
        else:
            parts[i] = 0
    else:
        # All positions at max_value, add new position
        parts.insert(0, 1)

    return "v" + ".".join(map(str, parts))
install_dependencies(yaml_file, auto=False)

Installs dependencies from tbConfig.yaml.

Parameters:

Name Type Description Default
yaml_file str

Path to configuration file

required
auto bool

Automatically install without confirmation

False

Returns:

Type Description
bool

True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
def install_dependencies(yaml_file: str, auto: bool = False) -> bool:
    """
    Installs dependencies from tbConfig.yaml.

    Args:
        yaml_file: Path to configuration file
        auto: Automatically install without confirmation

    Returns:
        True if successful, False otherwise
    """
    try:
        with open(yaml_file, 'r', encoding='utf-8') as f:
            config = yaml.safe_load(f)

        dependencies_file = config.get("dependencies_file")

        if not dependencies_file:
            print("⚠ No dependencies file specified")
            return True

        dependencies_path = Path(dependencies_file)

        if not dependencies_path.exists():
            print(f"⚠ Dependencies file not found: {dependencies_path}")
            return False

        print(f"Installing dependencies from: {dependencies_path}")

        if not auto:
            response = input("Continue with installation? (y/n): ")
            if response.lower() != 'y':
                print("Installation cancelled")
                return False

        subprocess.run(
            [sys.executable, '-m', 'pip', 'install', '-r', str(dependencies_path)],
            check=True
        )

        print("✓ Dependencies installed successfully")
        return True

    except Exception as e:
        print(f"✗ Error installing dependencies: {str(e)}")
        return False
install_from_zip(app, zip_name, no_dep=True, auto_dep=False, target_platform=None)

Installs a module from ZIP file with dependency management.

Parameters:

Name Type Description Default
app App

Application instance

required
zip_name str

Name of ZIP file

required
no_dep bool

Skip dependency installation

True
auto_dep bool

Automatically install dependencies

False
target_platform Optional[Platform]

Optional platform filter

None

Returns:

Type Description
bool

True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
def install_from_zip(app: App, zip_name: str, no_dep: bool = True,
                     auto_dep: bool = False,
                     target_platform: Optional[Platform] = None) -> bool:
    """
    Installs a module from ZIP file with dependency management.

    Args:
        app: Application instance
        zip_name: Name of ZIP file
        no_dep: Skip dependency installation
        auto_dep: Automatically install dependencies
        target_platform: Optional platform filter

    Returns:
        True if successful, False otherwise
    """
    zip_path = Path(app.start_dir) / "mods_sto" / zip_name

    if not zip_path.exists():
        print(f"✗ ZIP file not found: {zip_path}")
        return False

    try:
        with Spinner(f"Unpacking {zip_path.name[-40:]}"):
            module_name = unpack_and_move_module(
                str(zip_path),
                f"{app.start_dir}/mods",
                target_platform=target_platform
            )

        if not module_name:
            return False

        # Install dependencies if requested
        if not no_dep:
            config_path = Path(app.start_dir) / "mods" / module_name / "tbConfig.yaml"

            if config_path.exists():
                with Spinner(f"Installing dependencies for {module_name}"):
                    install_dependencies(str(config_path), auto_dep)

        return True

    except Exception as e:
        print(f"✗ Installation failed: {str(e)}")
        return False
installer(app, module_name, build_state=True, platform=None) async

Installs or updates a module from the server.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

required
module_name str

Name of module to install

required
build_state bool

Whether to rebuild state after installation

True
platform Optional[Platform]

Optional platform filter for installation

None

Returns:

Type Description
Result

Result with installation status

Source code in toolboxv2/mods/CloudM/ModManager.py
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
@export(mod_name=Name, name="install", test=False)
async def installer(app: Optional[App], module_name: str,
                    build_state: bool = True,
                    platform: Optional[Platform] = None) -> Result:
    """
    Installs or updates a module from the server.

    Args:
        app: Application instance
        module_name: Name of module to install
        build_state: Whether to rebuild state after installation
        platform: Optional platform filter for installation

    Returns:
        Result with installation status
    """
    if app is None:
        app = get_app(f"{Name}.install")

    if not app.session.valid and not await app.session.login():
        return Result.default_user_error("Please login with CloudM")

    try:
        # Get remote version
        response = await app.session.fetch(
            f"/api/{Name}/getModVersion?module_name={module_name}",
            method="GET"
        )
        remote_version = await response.text()
        remote_version = None if remote_version == "None" else remote_version.strip('"')

        # Get local version
        local_version = find_highest_zip_version(module_name, version_only=True)

        if not local_version and not remote_version:
            return Result.default_user_error(f"Module '{module_name}' not found (404)")

        # Compare versions
        local_ver = pv.parse(local_version) if local_version else pv.parse("0.0.0")
        remote_ver = pv.parse(remote_version) if remote_version else pv.parse("0.0.0")

        app.print(f"Module versions - Local: {local_ver}, Remote: {remote_ver}")

        if remote_ver > local_ver:
            download_path = Path(app.start_dir) / 'mods_sto'
            download_url = f"/api/{Name}/download_mod?module_name={module_name}"

            if platform:
                download_url += f"&platform={platform.value}"

            app.print(f"Downloading from {app.session.base}{download_url}")

            if not await app.session.download_file(download_url, str(download_path)):
                app.print("⚠ Automatic download failed")
                manual = input("Download manually and place in mods_sto folder. Done? (y/n): ")
                if 'y' not in manual.lower():
                    return Result.default_user_error("Installation cancelled")

            zip_name = f"RST${module_name}&{app.version}§{remote_version}.zip"

            with Spinner("Installing from ZIP"):
                success = install_from_zip(app, zip_name, target_platform=platform)

            if not success:
                return Result.default_internal_error("Installation failed")

            if build_state:
                with Spinner("Rebuilding state"):
                    get_state_from_app(app)

            return Result.ok({
                "message": f"Module '{module_name}' installed successfully",
                "version": remote_version
            })

        app.print("✓ Module is already up to date")
        return Result.ok("Module is up to date")

    except Exception as e:
        return Result.default_internal_error(f"Installation failed: {str(e)}")
interactive_manager(app=None) async

Modern interactive CLI manager for module operations.

Features: - Arrow key navigation (↑↓ or w/s) - Modern, minimalistic UI - All original functionality preserved - Better visual feedback - Elegant dialogs and prompts

Source code in toolboxv2/mods/CloudM/ModManager.py
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
@export(mod_name=Name, name="manager", test=False)
async def interactive_manager(app: Optional[App] = None):
    """
    Modern interactive CLI manager for module operations.

    Features:
    - Arrow key navigation (↑↓ or w/s)
    - Modern, minimalistic UI
    - All original functionality preserved
    - Better visual feedback
    - Elegant dialogs and prompts
    """
    if app is None:
        app = get_app(f"{Name}.manager")

    # Clear screen
    print('\033[2J\033[H')

    # Welcome message
    print_formatted_text(HTML(
        '\n<menu-title>╔════════════════════════════════════════════════════════════════════╗</menu-title>\n'
        '<menu-title>║          Welcome to CloudM Interactive Module Manager              ║</menu-title>\n'
        '<menu-title>╚════════════════════════════════════════════════════════════════════╝</menu-title>\n'
    ), style=MODERN_STYLE)

    await asyncio.sleep(1)

    # Create and run manager
    manager = ModernMenuManager(app)

    try:
        await manager.run()
    except KeyboardInterrupt:
        pass
    finally:
        # Goodbye message
        print('\033[2J\033[H')
        print_formatted_text(HTML(
            '\n<success>╔════════════════════════════════════════════════════════════════════╗</success>\n'
            '<success>║          Thank you for using CloudM Module Manager! 👋             ║</success>\n'
            '<success>╚════════════════════════════════════════════════════════════════════╝</success>\n'
        ), style=MODERN_STYLE)
list_module_templates(app=None)

Lists all available module templates.

Returns:

Type Description
Result

Result with template information

Source code in toolboxv2/mods/CloudM/ModManager.py
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
@export(mod_name=Name, name="list_templates", test=False)
def list_module_templates(app: Optional[App] = None) -> Result:
    """
    Lists all available module templates.

    Returns:
        Result with template information
    """
    templates = []
    for template_name, template_info in MODULE_TEMPLATES.items():
        templates.append({
            "name": template_name,
            "description": template_info["description"],
            "type": template_info["type"],
            "requires": template_info["requires"]
        })

    return Result.ok(data={"templates": templates, "count": len(templates)})
list_modules(app=None)

Lists all available modules.

Returns:

Type Description
Result

Result with module list

Source code in toolboxv2/mods/CloudM/ModManager.py
797
798
799
800
801
802
803
804
805
806
807
808
809
@export(mod_name=Name, api=True, interface=ToolBoxInterfaces.remote, test=False)
def list_modules(app: App = None) -> Result:
    """
    Lists all available modules.

    Returns:
        Result with module list
    """
    if app is None:
        app = get_app("cm.list_modules")

    modules = app.get_all_mods()
    return Result.ok({"modules": modules, "count": len(modules)})
load_and_validate_config(config_path)

Loads and validates a configuration file.

Parameters:

Name Type Description Default
config_path Path

Path to configuration file

required

Returns:

Type Description
Tuple[Optional[Dict], List[str]]

Tuple of (config_dict or None, list_of_errors)

Source code in toolboxv2/mods/CloudM/ModManager.py
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
def load_and_validate_config(config_path: Path) -> Tuple[Optional[Dict], List[str]]:
    """
    Loads and validates a configuration file.

    Args:
        config_path: Path to configuration file

    Returns:
        Tuple of (config_dict or None, list_of_errors)
    """
    if not config_path.exists():
        return None, [f"Config file not found: {config_path}"]

    try:
        with open(config_path, 'r', encoding='utf-8') as f:
            config = yaml.safe_load(f)
    except Exception as e:
        return None, [f"Failed to parse YAML: {str(e)}"]

    # Determine schema based on module_type
    module_type = config.get("module_type", "package")

    if module_type == "single":
        schema = TB_CONFIG_SINGLE_SCHEMA
    else:
        schema = TB_CONFIG_SCHEMA_V2

    is_valid, errors = validate_config(config, schema)

    if not is_valid:
        return config, errors

    return config, []
make_installer(app, module_name, base='./mods', upload=None, platform=None) async

Creates an installer package for a module.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

required
module_name str

Name of module to package

required
base str

Base directory containing modules

'./mods'
upload Optional[bool]

Whether to upload after creation

None
platform Optional[Platform]

Optional platform filter

None

Returns:

Type Description
Result

Result with package path or upload status

Source code in toolboxv2/mods/CloudM/ModManager.py
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
@export(mod_name=Name, name="make_install", test=False)
async def make_installer(app: Optional[App], module_name: str,
                         base: str = "./mods", upload: Optional[bool] = None,
                         platform: Optional[Platform] = None) -> Result:
    """
    Creates an installer package for a module.

    Args:
        app: Application instance
        module_name: Name of module to package
        base: Base directory containing modules
        upload: Whether to upload after creation
        platform: Optional platform filter

    Returns:
        Result with package path or upload status
    """
    if app is None:
        app = get_app(f"{Name}.make_install")

    if module_name not in app.get_all_mods():
        return Result.default_user_error(f"Module '{module_name}' not found")

    try:
        with Spinner("Testing module load"):
            app.save_load(module_name)

        mod = app.get_mod(module_name)
        version_ = getattr(mod, 'version', version)

        with Spinner("Creating and packing module"):
            zip_path = create_and_pack_module(
                base, module_name, version_,
                platform_filter=platform
            )

        if not zip_path:
            return Result.default_internal_error("Failed to create package")

        # Upload if requested
        if upload or (upload is None and 'y' in input("Upload ZIP file? (y/n): ").lower()):
            with Spinner("Uploading file"):
                res = await app.session.upload_file(zip_path, '/installer/upload-file/')

            if isinstance(res, dict):
                if res.get('res', '').startswith('Successfully uploaded'):
                    return Result.ok({
                        "message": "Module packaged and uploaded",
                        "zip_path": zip_path,
                        "upload_response": res
                    })
                return Result.default_user_error(res)

        return Result.ok({
            "message": "Module packaged successfully",
            "zip_path": zip_path
        })

    except Exception as e:
        return Result.default_internal_error(f"Installation creation failed: {str(e)}")
mod_manager_ui(app)

Serves the module manager web interface.

Returns:

Type Description
Result

HTML result with UI

Source code in toolboxv2/mods/CloudM/ModManager.py
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
@export(mod_name=Name, name="ui", api=True, api_methods=['GET'])
def mod_manager_ui(app: App) -> Result:
    """
    Serves the module manager web interface.

    Returns:
        HTML result with UI
    """
    ui_path = Path(__file__).parent / "mod_manager.html"

    if not ui_path.exists():
        # Generate default UI if file doesn't exist
        html_content = """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>CloudM - Module Manager</title>
    <style>
        * { margin: 0; padding: 0; box-sizing: border-box; }
        body {
            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px;
        }
        .container {
            max-width: 1200px;
            margin: 0 auto;
            background: white;
            border-radius: 10px;
            box-shadow: 0 20px 60px rgba(0,0,0,0.3);
            overflow: hidden;
        }
        .header {
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            text-align: center;
        }
        .content {
            padding: 30px;
        }
        .module-list {
            display: grid;
            grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
            gap: 20px;
            margin-top: 20px;
        }
        .module-card {
            border: 1px solid #e0e0e0;
            border-radius: 8px;
            padding: 20px;
            transition: transform 0.2s, box-shadow 0.2s;
        }
        .module-card:hover {
            transform: translateY(-5px);
            box-shadow: 0 5px 20px rgba(0,0,0,0.1);
        }
        .btn {
            background: #667eea;
            color: white;
            border: none;
            padding: 10px 20px;
            border-radius: 5px;
            cursor: pointer;
            margin: 5px;
            transition: background 0.3s;
        }
        .btn:hover { background: #5568d3; }
        .btn-danger { background: #e74c3c; }
        .btn-danger:hover { background: #c0392b; }
    </style>
</head>
<body>
    <div class="container">
        <div class="header">
            <h1>🚀 CloudM Module Manager</h1>
            <p>Manage your modules with ease</p>
        </div>
        <div class="content">
            <div class="controls">
                <button class="btn" onclick="loadModules()">🔄 Refresh</button>
                <button class="btn" onclick="updateAll()">⬆️ Update All</button>
            </div>
            <div id="modules" class="module-list"></div>
        </div>
    </div>
    <script>
        async function loadModules() {
            const response = await fetch('/api/CloudM/list_modules');
            const data = await response.json();
            const container = document.getElementById('modules');
            container.innerHTML = data.modules.map(mod => `
                <div class="module-card">
                    <h3>📦 ${mod}</h3>
                    <button class="btn" onclick="installModule('${mod}')">Install</button>
                    <button class="btn btn-danger" onclick="uninstallModule('${mod}')">Uninstall</button>
                </div>
            `).join('');
        }
        async function installModule(name) {
            alert(`Installing ${name}...`);
        }
        async function uninstallModule(name) {
            if (confirm(`Uninstall ${name}?`)) {
                alert(`Uninstalling ${name}...`);
            }
        }
        async function updateAll() {
            alert('Updating all modules...');
        }
        loadModules();
    </script>
</body>
</html>
        """
        return Result.html(html_content)

    return Result.html(ui_path.read_text(encoding='utf-8'))
run_command(command, cwd=None)

Executes a command and returns output.

Parameters:

Name Type Description Default
command List[str]

Command and arguments as list

required
cwd Optional[str]

Working directory for command execution

None

Returns:

Type Description
str

Command stdout output

Raises:

Type Description
CalledProcessError

If command fails

Source code in toolboxv2/mods/CloudM/ModManager.py
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
def run_command(command: List[str], cwd: Optional[str] = None) -> str:
    """
    Executes a command and returns output.

    Args:
        command: Command and arguments as list
        cwd: Working directory for command execution

    Returns:
        Command stdout output

    Raises:
        subprocess.CalledProcessError: If command fails
    """
    result = subprocess.run(
        command,
        cwd=cwd,
        capture_output=True,
        text=True,
        check=True,
        encoding='utf-8'
    )
    return result.stdout
show_choice(title, text, choices) async

Show radio list dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py
1420
1421
1422
1423
1424
1425
1426
1427
1428
async def show_choice(title: str, text: str, choices: List[tuple]) -> Optional[Any]:
    """Show radio list dialog."""
    result = await radiolist_dialog(
        title=f"◉ {title}",
        text=text,
        values=choices,
        style=MODERN_STYLE
    ).run_async()
    return result
show_confirm(title, text) async

Show confirmation dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py
1399
1400
1401
1402
1403
1404
1405
1406
async def show_confirm(title: str, text: str) -> bool:
    """Show confirmation dialog."""
    result = await yes_no_dialog(
        title=f"⚠ {title}",
        text=text,
        style=MODERN_STYLE
    ).run_async()
    return result if result is not None else False
show_input(title, label, default='') async

Show input dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py
1409
1410
1411
1412
1413
1414
1415
1416
1417
async def show_input(title: str, label: str, default: str = "") -> Optional[str]:
    """Show input dialog."""
    result = await input_dialog(
        title=f"✎ {title}",
        text=label,
        default=default,
        style=MODERN_STYLE
    ).run_async()
    return result
show_message(title, text, style='info') async

Show a message dialog.

Source code in toolboxv2/mods/CloudM/ModManager.py
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
async def show_message(title: str, text: str, style: str = "info"):
    """Show a message dialog."""
    icons = {
        'success': '✓',
        'error': '✗',
        'warning': '⚠',
        'info': 'ℹ'
    }
    icon = icons.get(style, 'ℹ')

    await message_dialog(
        title=f"{icon} {title}",
        text=text,
        style=MODERN_STYLE
    ).run_async()
show_progress(title, message) async

Show a simple progress message.

Source code in toolboxv2/mods/CloudM/ModManager.py
1431
1432
1433
1434
async def show_progress(title: str, message: str):
    """Show a simple progress message."""
    print_formatted_text(HTML(f'\n<info>⟳ {title}</info>'))
    print_formatted_text(HTML(f'<menu-item>  {message}</menu-item>\n'))
uninstall_dependencies(yaml_file)

Uninstalls dependencies from tbConfig.yaml.

Parameters:

Name Type Description Default
yaml_file str

Path to configuration file

required

Returns:

Type Description
bool

True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
def uninstall_dependencies(yaml_file: str) -> bool:
    """
    Uninstalls dependencies from tbConfig.yaml.

    Args:
        yaml_file: Path to configuration file

    Returns:
        True if successful, False otherwise
    """
    try:
        with open(yaml_file, 'r', encoding='utf-8') as f:
            config = yaml.safe_load(f)

        dependencies = config.get("dependencies", [])

        if not dependencies:
            print("⚠ No dependencies to uninstall")
            return True

        for dependency in dependencies:
            print(f"Uninstalling: {dependency}")
            subprocess.run(
                [sys.executable, '-m', 'pip', 'uninstall', '-y', dependency],
                check=True
            )

        print("✓ Dependencies uninstalled successfully")
        return True

    except Exception as e:
        print(f"✗ Error uninstalling dependencies: {str(e)}")
        return False
uninstall_module(path, module_name='', version='-.-.-', additional_dirs=None, yaml_data=None)

Uninstalls a module by removing its directory and ZIP file.

Parameters:

Name Type Description Default
path str

Base path containing module

required
module_name str

Name of module to uninstall

''
version str

Module version

'-.-.-'
additional_dirs Optional[Dict]

Additional directories to remove

None
yaml_data Optional[Dict]

Configuration data

None

Returns:

Type Description
bool

True if successful, False otherwise

Source code in toolboxv2/mods/CloudM/ModManager.py
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
def uninstall_module(path: str, module_name: str = '', version: str = '-.-.-',
                     additional_dirs: Optional[Dict] = None,
                     yaml_data: Optional[Dict] = None) -> bool:
    """
    Uninstalls a module by removing its directory and ZIP file.

    Args:
        path: Base path containing module
        module_name: Name of module to uninstall
        version: Module version
        additional_dirs: Additional directories to remove
        yaml_data: Configuration data

    Returns:
        True if successful, False otherwise
    """
    if additional_dirs is None:
        additional_dirs = {}

    base_path = Path(path).parent
    module_path = base_path / module_name
    zip_path = Path(f"./mods_sto/RST${module_name}&{__version__}§{version}.zip")

    if not module_path.exists():
        print(f"⚠ Module {module_name} already uninstalled")
        return False

    try:
        # Remove module directory
        shutil.rmtree(module_path)
        print(f"✓ Removed module directory: {module_path}")

        # Remove additional directories
        for _dir_name, dir_paths in additional_dirs.items():
            if isinstance(dir_paths, str):
                dir_paths = [dir_paths]

            for dir_path in dir_paths:
                dir_path = Path(dir_path)
                if dir_path.exists():
                    shutil.rmtree(dir_path)
                    print(f"✓ Removed additional path: {dir_path}")

        # Remove ZIP file
        if zip_path.exists():
            zip_path.unlink()
            print(f"✓ Removed ZIP file: {zip_path}")

        return True

    except Exception as e:
        print(f"✗ Error during uninstallation: {str(e)}")
        return False
uninstaller(app, module_name)

Uninstalls a module.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

required
module_name str

Name of module to uninstall

required

Returns:

Type Description
Result

Result with uninstallation status

Source code in toolboxv2/mods/CloudM/ModManager.py
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
@export(mod_name=Name, name="uninstall", test=False)
def uninstaller(app: Optional[App], module_name: str) -> Result:
    """
    Uninstalls a module.

    Args:
        app: Application instance
        module_name: Name of module to uninstall

    Returns:
        Result with uninstallation status
    """
    if app is None:
        app = get_app(f"{Name}.uninstall")

    if module_name not in app.get_all_mods():
        return Result.default_user_error(f"Module '{module_name}' not found")

    try:
        mod = app.get_mod(module_name)
        version_ = getattr(mod, 'version', version)

        confirm = input(f"Uninstall module '{module_name}' v{version_}? (y/n): ")
        if 'y' not in confirm.lower():
            return Result.ok("Uninstallation cancelled")

        success = uninstall_module(f"./mods/{module_name}", module_name, version_)

        if success:
            return Result.ok(f"Module '{module_name}' uninstalled successfully")
        else:
            return Result.default_internal_error("Uninstallation failed")

    except Exception as e:
        return Result.default_internal_error(f"Uninstallation failed: {str(e)}")
unpack_and_move_module(zip_path, base_path='./mods', module_name='', target_platform=None)

Unpacks a ZIP file and moves contents with platform filtering.

Parameters:

Name Type Description Default
zip_path str

Path to ZIP file

required
base_path str

Base installation path

'./mods'
module_name str

Module name (extracted from ZIP if not provided)

''
target_platform Optional[Platform]

Optional platform filter for installation

None

Returns:

Type Description
Optional[str]

Name of installed module or None on failure

Source code in toolboxv2/mods/CloudM/ModManager.py
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
def unpack_and_move_module(zip_path: str, base_path: str = './mods',
                           module_name: str = '',
                           target_platform: Optional[Platform] = None) -> Optional[str]:
    """
    Unpacks a ZIP file and moves contents with platform filtering.

    Args:
        zip_path: Path to ZIP file
        base_path: Base installation path
        module_name: Module name (extracted from ZIP if not provided)
        target_platform: Optional platform filter for installation

    Returns:
        Name of installed module or None on failure
    """
    zip_path = Path(zip_path)
    base_path = Path(base_path)

    if not module_name:
        module_name = zip_path.name.split('$')[1].split('&')[0]

    module_path = base_path / module_name
    temp_base = Path('./mods_sto/temp')

    try:
        temp_base.mkdir(parents=True, exist_ok=True)

        with tempfile.TemporaryDirectory(dir=str(temp_base)) as temp_dir:
            temp_dir = Path(temp_dir)

            with Spinner(f"Extracting {zip_path.name}"):
                with zipfile.ZipFile(zip_path, 'r') as zip_ref:
                    zip_ref.extractall(temp_dir)

            # Load configuration to check for platform-specific installation
            config_path = temp_dir / module_name / "tbConfig.yaml"
            if not config_path.exists():
                config_path = temp_dir / f"{module_name}.yaml"

            config, errors = load_and_validate_config(config_path)

            if errors:
                print(f"⚠ Configuration validation warnings: {', '.join(errors)}")

            # Handle module directory
            source_module = temp_dir / module_name

            if source_module.exists():
                with Spinner(f"Installing module to {module_path}"):
                    if module_path.exists():
                        shutil.rmtree(module_path)

                    # If platform filtering is enabled and config exists
                    if target_platform and config:
                        platform_files = get_platform_files(config, target_platform)

                        # Install only platform-specific files
                        module_path.mkdir(parents=True, exist_ok=True)

                        for pattern in platform_files:
                            if pattern == "*":
                                # Copy all files
                                shutil.copytree(source_module, module_path, dirs_exist_ok=True)
                                break
                            else:
                                # Copy specific files/patterns
                                for file in source_module.glob(pattern):
                                    if file.is_file():
                                        shutil.copy2(file, module_path)
                                    elif file.is_dir():
                                        shutil.copytree(file, module_path / file.name, dirs_exist_ok=True)
                    else:
                        # Install all files
                        shutil.copytree(source_module, module_path, dirs_exist_ok=True)

            # Handle additional files in root
            with Spinner("Installing additional files"):
                for item in temp_dir.iterdir():
                    if item.name == module_name or item.name.endswith('.yaml'):
                        continue

                    target = Path('./') / item.name
                    if item.is_dir():
                        if target.exists():
                            shutil.rmtree(target)
                        shutil.copytree(item, target, dirs_exist_ok=True)
                    else:
                        shutil.copy2(item, target)

            print(f"✓ Successfully installed/updated module: {module_name}")
            return module_name

    except Exception as e:
        print(f"✗ Error during installation: {str(e)}")
        if module_path.exists():
            shutil.rmtree(module_path)
        raise
update_all_mods(app) async

Updates all installed modules.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

required

Returns:

Type Description
Result

Result with update summary

Source code in toolboxv2/mods/CloudM/ModManager.py
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
@export(mod_name=Name, name="update_all", test=False)
async def update_all_mods(app: Optional[App]) -> Result:
    """
    Updates all installed modules.

    Args:
        app: Application instance

    Returns:
        Result with update summary
    """
    if app is None:
        app = get_app(f"{Name}.update_all")

    all_mods = app.get_all_mods()
    results = {"updated": [], "failed": [], "up_to_date": []}

    async def check_and_update(mod_name: str):
        try:
            # Get remote version
            response = await app.session.fetch(
                f"/api/{Name}/getModVersion?module_name={mod_name}"
            )
            remote_version = await response.text()
            remote_version = remote_version.strip('"') if remote_version != "None" else None

            if not remote_version:
                results["failed"].append({"module": mod_name, "reason": "Version not found"})
                return

            local_mod = app.get_mod(mod_name)
            if not local_mod:
                results["failed"].append({"module": mod_name, "reason": "Local module not found"})
                return

            local_version = getattr(local_mod, 'version', '0.0.0')

            if pv.parse(remote_version) > pv.parse(local_version):
                result = await installer(app, mod_name, build_state=False)
                if result.is_error:
                    results["failed"].append({"module": mod_name, "reason": str(result)})
                else:
                    results["updated"].append({"module": mod_name, "version": remote_version})
            else:
                results["up_to_date"].append(mod_name)

        except Exception as e:
            results["failed"].append({"module": mod_name, "reason": str(e)})

    # Run updates in parallel
    await asyncio.gather(*[check_and_update(mod) for mod in all_mods])

    # Rebuild state once at the end
    with Spinner("Rebuilding application state"):
        get_state_from_app(app)

    return Result.ok({
        "summary": {
            "total": len(all_mods),
            "updated": len(results["updated"]),
            "up_to_date": len(results["up_to_date"]),
            "failed": len(results["failed"])
        },
        "details": results
    })
upload(app, module_name) async

Uploads an existing module package to the server.

Parameters:

Name Type Description Default
app Optional[App]

Application instance

required
module_name str

Name of module to upload

required

Returns:

Type Description
Result

Result with upload status

Source code in toolboxv2/mods/CloudM/ModManager.py
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
@export(mod_name=Name, name="upload", test=False)
async def upload(app: Optional[App], module_name: str) -> Result:
    """
    Uploads an existing module package to the server.

    Args:
        app: Application instance
        module_name: Name of module to upload

    Returns:
        Result with upload status
    """
    if app is None:
        app = get_app(f"{Name}.upload")

    try:
        zip_path = find_highest_zip_version(module_name)

        if not zip_path:
            return Result.default_user_error(f"No package found for module '{module_name}'")

        confirm = input(f"Upload ZIP file {zip_path}? (y/n): ")
        if 'y' not in confirm.lower():
            return Result.ok("Upload cancelled")

        res = await app.session.upload_file(zip_path, f'/api/{Name}/upload_mod')

        return Result.ok({
            "message": "Upload completed",
            "response": res
        })

    except Exception as e:
        return Result.default_internal_error(f"Upload failed: {str(e)}")
upload_mod(app, request, form_data=None) async

Uploads a module ZIP file to the server.

Parameters:

Name Type Description Default
app App

Application instance

required
request RequestData

Request data

required
form_data Optional[Dict[str, Any]]

Form data containing file

None

Returns:

Type Description
Result

Result with upload status

Source code in toolboxv2/mods/CloudM/ModManager.py
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
@export(mod_name=Name, name="upload_mod", api=True, api_methods=['POST'])
async def upload_mod(app: App, request: RequestData,
                     form_data: Optional[Dict[str, Any]] = None) -> Result:
    """
    Uploads a module ZIP file to the server.

    Args:
        app: Application instance
        request: Request data
        form_data: Form data containing file

    Returns:
        Result with upload status
    """
    if not isinstance(form_data, dict):
        return Result.default_user_error("No form data provided")

    if form_data is None or 'files' not in form_data:
        return Result.default_user_error("No file provided")

    try:
        uploaded_file = form_data.get('files')[0]
        file_name = uploaded_file.filename
        file_bytes = uploaded_file.file.read()

        # Security validation
        if not file_name.endswith('.zip'):
            return Result.default_user_error("Only ZIP files are allowed")

        if not file_name.startswith('RST$'):
            return Result.default_user_error("Invalid module ZIP format")

        # Save file
        save_path = Path(app.start_dir) / "mods_sto" / file_name
        save_path.parent.mkdir(parents=True, exist_ok=True)
        save_path.write_bytes(file_bytes)

        # Extract module info
        module_name = file_name.split('$')[1].split('&')[0]
        module_version = file_name.split('§')[1].replace('.zip', '')

        return Result.ok({
            "message": f"Successfully uploaded {file_name}",
            "module": module_name,
            "version": module_version,
            "size": len(file_bytes)
        })

    except Exception as e:
        return Result.default_internal_error(f"Upload failed: {str(e)}")
validate_config(config, schema)

Validates configuration against schema.

Parameters:

Name Type Description Default
config Dict

Configuration dictionary to validate

required
schema Dict

Schema dictionary with expected types

required

Returns:

Type Description
Tuple[bool, List[str]]

Tuple of (is_valid, list_of_errors)

Source code in toolboxv2/mods/CloudM/ModManager.py
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def validate_config(config: Dict, schema: Dict) -> Tuple[bool, List[str]]:
    """
    Validates configuration against schema.

    Args:
        config: Configuration dictionary to validate
        schema: Schema dictionary with expected types

    Returns:
        Tuple of (is_valid, list_of_errors)
    """
    errors = []

    def check_type(key: str, value: Any, expected_type: Any, path: str = ""):
        full_path = f"{path}.{key}" if path else key

        if isinstance(expected_type, dict):
            if not isinstance(value, dict):
                errors.append(f"{full_path}: Expected dict, got {type(value).__name__}")
                return
            for sub_key, sub_type in expected_type.items():
                if sub_key in value:
                    check_type(sub_key, value[sub_key], sub_type, full_path)
        elif expected_type == list:
            if not isinstance(value, list):
                errors.append(f"{full_path}: Expected list, got {type(value).__name__}")
        elif expected_type == dict:
            if not isinstance(value, dict):
                errors.append(f"{full_path}: Expected dict, got {type(value).__name__}")
        elif not isinstance(value, expected_type):
            errors.append(f"{full_path}: Expected {expected_type.__name__}, got {type(value).__name__}")

    for key, expected_type in schema.items():
        if key in config:
            check_type(key, config[key], expected_type)

    return len(errors) == 0, errors

ModManager_tests

TestModManager

Bases: TestCase

Source code in toolboxv2/mods/CloudM/ModManager_tests.py
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
class TestModManager(unittest.TestCase):
    app: App = None

    def test_increment_version(self):
        """Tests the version increment logic."""
        print("\nTesting increment_version...")
        self.assertEqual(increment_version("v0.0.1"), "v0.0.2")
        self.assertEqual(increment_version("v0.0.99", max_value=99), "v0.1.0")
        self.assertEqual(increment_version("v0.99.99", max_value=99), "v1.0.0")
        self.assertEqual(increment_version("v98"), "v99")
        with self.assertRaises(ValueError, msg="Should fail if 'v' is missing"):
            print(increment_version("0.0.1"))
        print("increment_version tests passed.")

    def setUp(self):
        """Set up a temporary environment for each test."""
        self.original_cwd = os.getcwd()
        self.test_dir = tempfile.mkdtemp(prefix="mod_manager_test_")

        # The functions in ModManager use relative paths like './mods' and './mods_sto'
        # We'll create these inside our temp directory and chdir into it.
        os.chdir(self.test_dir)
        os.makedirs("mods", exist_ok=True)
        os.makedirs("mods_sto", exist_ok=True)
        os.makedirs("source_module", exist_ok=True)

    def tearDown(self):
        """Clean up the temporary environment after each test."""
        os.chdir(self.original_cwd)
        shutil.rmtree(self.test_dir, ignore_errors=True)

    def test_create_pack_unpack_cycle(self):
        """Tests the full cycle of creating, packing, and unpacking a module."""
        print("\nTesting create_pack_unpack_cycle...")
        module_name = "MyTestMod"
        module_version = "v0.1.0"

        # 1. Create a dummy module structure inside the temp 'source_module' dir
        source_path = Path("source_module")
        module_source_path = source_path / module_name
        module_source_path.mkdir()
        (module_source_path / "main.py").write_text("print('hello from my test mod')")
        (module_source_path / "data.txt").write_text("some test data")

        # 2. Call create_and_pack_module
        # The 'path' argument is the parent directory of the module directory.
        zip_path_str = create_and_pack_module(
            path=str(source_path),
            module_name=module_name,
            version=module_version
        )
        self.assertTrue(zip_path_str, "create_and_pack_module should return a path.")
        zip_path = Path(zip_path_str)

        # 3. Assert the zip file was created in the correct location ('./mods_sto')
        self.assertTrue(zip_path.exists(), f"Zip file should exist at {zip_path}")
        self.assertEqual(zip_path.parent.name, "mods_sto")

        # 4. Call unpack_and_move_module
        # We unpack into the './mods' directory.
        unpacked_name = unpack_and_move_module(
            zip_path=str(zip_path),
            base_path="mods"
        )

        # 5. Assert the module was unpacked correctly
        self.assertEqual(unpacked_name, module_name)
        unpacked_dir = Path("mods") / module_name
        self.assertTrue(unpacked_dir.is_dir(), "Unpacked module directory should exist.")

        # Verify content
        self.assertTrue((unpacked_dir / "main.py").exists())
        self.assertEqual((unpacked_dir / "main.py").read_text(), "print('hello from my test mod')")
        self.assertTrue((unpacked_dir / "data.txt").exists())
        self.assertEqual((unpacked_dir / "data.txt").read_text(), "some test data")

        # Verify that the tbConfig.yaml was created and has correct info
        config_path = unpacked_dir / "tbConfig.yaml"
        self.assertTrue(config_path.exists())
        with open(config_path) as f:
            config = yaml.safe_load(f)
        self.assertEqual(config.get("module_name"), module_name)
        self.assertEqual(config.get("version"), module_version)

        print("create_pack_unpack_cycle tests passed.")

    def test_install_from_zip(self):
        """Tests the install_from_zip helper function."""
        print("\nTesting install_from_zip...")
        module_name = "MyInstallTestMod"
        module_version = "v0.1.1"

        # 1. Create a dummy module and zip it
        source_path = Path("source_module")
        module_source_path = source_path / module_name
        module_source_path.mkdir()
        (module_source_path / "main.py").write_text("pass")
        zip_path_str = create_and_pack_module(
            path=str(source_path),
            module_name=module_name,
            version=module_version
        )
        zip_path = Path(zip_path_str)
        zip_name = zip_path.name

        # 2. Mock the app object needed by install_from_zip
        mock_app = lambda :None
        mock_app.start_dir = self.test_dir

        # 3. Call install_from_zip
        result = install_from_zip(mock_app, zip_name, no_dep=True)

        # 4. Assert the installation was successful
        self.assertTrue(result)
        unpacked_dir = Path("mods") / module_name
        self.assertTrue(unpacked_dir.is_dir())
        self.assertTrue((unpacked_dir / "main.py").exists())
        print("install_from_zip tests passed.")
setUp()

Set up a temporary environment for each test.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py
58
59
60
61
62
63
64
65
66
67
68
def setUp(self):
    """Set up a temporary environment for each test."""
    self.original_cwd = os.getcwd()
    self.test_dir = tempfile.mkdtemp(prefix="mod_manager_test_")

    # The functions in ModManager use relative paths like './mods' and './mods_sto'
    # We'll create these inside our temp directory and chdir into it.
    os.chdir(self.test_dir)
    os.makedirs("mods", exist_ok=True)
    os.makedirs("mods_sto", exist_ok=True)
    os.makedirs("source_module", exist_ok=True)
tearDown()

Clean up the temporary environment after each test.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py
70
71
72
73
def tearDown(self):
    """Clean up the temporary environment after each test."""
    os.chdir(self.original_cwd)
    shutil.rmtree(self.test_dir, ignore_errors=True)
test_create_pack_unpack_cycle()

Tests the full cycle of creating, packing, and unpacking a module.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def test_create_pack_unpack_cycle(self):
    """Tests the full cycle of creating, packing, and unpacking a module."""
    print("\nTesting create_pack_unpack_cycle...")
    module_name = "MyTestMod"
    module_version = "v0.1.0"

    # 1. Create a dummy module structure inside the temp 'source_module' dir
    source_path = Path("source_module")
    module_source_path = source_path / module_name
    module_source_path.mkdir()
    (module_source_path / "main.py").write_text("print('hello from my test mod')")
    (module_source_path / "data.txt").write_text("some test data")

    # 2. Call create_and_pack_module
    # The 'path' argument is the parent directory of the module directory.
    zip_path_str = create_and_pack_module(
        path=str(source_path),
        module_name=module_name,
        version=module_version
    )
    self.assertTrue(zip_path_str, "create_and_pack_module should return a path.")
    zip_path = Path(zip_path_str)

    # 3. Assert the zip file was created in the correct location ('./mods_sto')
    self.assertTrue(zip_path.exists(), f"Zip file should exist at {zip_path}")
    self.assertEqual(zip_path.parent.name, "mods_sto")

    # 4. Call unpack_and_move_module
    # We unpack into the './mods' directory.
    unpacked_name = unpack_and_move_module(
        zip_path=str(zip_path),
        base_path="mods"
    )

    # 5. Assert the module was unpacked correctly
    self.assertEqual(unpacked_name, module_name)
    unpacked_dir = Path("mods") / module_name
    self.assertTrue(unpacked_dir.is_dir(), "Unpacked module directory should exist.")

    # Verify content
    self.assertTrue((unpacked_dir / "main.py").exists())
    self.assertEqual((unpacked_dir / "main.py").read_text(), "print('hello from my test mod')")
    self.assertTrue((unpacked_dir / "data.txt").exists())
    self.assertEqual((unpacked_dir / "data.txt").read_text(), "some test data")

    # Verify that the tbConfig.yaml was created and has correct info
    config_path = unpacked_dir / "tbConfig.yaml"
    self.assertTrue(config_path.exists())
    with open(config_path) as f:
        config = yaml.safe_load(f)
    self.assertEqual(config.get("module_name"), module_name)
    self.assertEqual(config.get("version"), module_version)

    print("create_pack_unpack_cycle tests passed.")
test_increment_version()

Tests the version increment logic.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py
47
48
49
50
51
52
53
54
55
56
def test_increment_version(self):
    """Tests the version increment logic."""
    print("\nTesting increment_version...")
    self.assertEqual(increment_version("v0.0.1"), "v0.0.2")
    self.assertEqual(increment_version("v0.0.99", max_value=99), "v0.1.0")
    self.assertEqual(increment_version("v0.99.99", max_value=99), "v1.0.0")
    self.assertEqual(increment_version("v98"), "v99")
    with self.assertRaises(ValueError, msg="Should fail if 'v' is missing"):
        print(increment_version("0.0.1"))
    print("increment_version tests passed.")
test_install_from_zip()

Tests the install_from_zip helper function.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
def test_install_from_zip(self):
    """Tests the install_from_zip helper function."""
    print("\nTesting install_from_zip...")
    module_name = "MyInstallTestMod"
    module_version = "v0.1.1"

    # 1. Create a dummy module and zip it
    source_path = Path("source_module")
    module_source_path = source_path / module_name
    module_source_path.mkdir()
    (module_source_path / "main.py").write_text("pass")
    zip_path_str = create_and_pack_module(
        path=str(source_path),
        module_name=module_name,
        version=module_version
    )
    zip_path = Path(zip_path_str)
    zip_name = zip_path.name

    # 2. Mock the app object needed by install_from_zip
    mock_app = lambda :None
    mock_app.start_dir = self.test_dir

    # 3. Call install_from_zip
    result = install_from_zip(mock_app, zip_name, no_dep=True)

    # 4. Assert the installation was successful
    self.assertTrue(result)
    unpacked_dir = Path("mods") / module_name
    self.assertTrue(unpacked_dir.is_dir())
    self.assertTrue((unpacked_dir / "main.py").exists())
    print("install_from_zip tests passed.")
run_mod_manager_tests(app)

This function will be automatically discovered and run by the test runner. It uses the standard unittest framework to run tests.

Source code in toolboxv2/mods/CloudM/ModManager_tests.py
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
@export(test_only=True)
def run_mod_manager_tests(app: App):
    """
    This function will be automatically discovered and run by the test runner.
    It uses the standard unittest framework to run tests.
    """
    print("Running ModManager Tests...")
    # We pass the app instance to the test class so it can be used if needed.
    TestModManager.app = app
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(TestModManager))
    runner = unittest.TextTestRunner()
    result = runner.run(suite)
    if not result.wasSuccessful():
        # Raise an exception to signal failure to the toolboxv2 test runner
        raise AssertionError(f"ModManager tests failed: {result.errors} {result.failures}")
    print("ModManager tests passed successfully.")
    return True

UserAccountManager

get_current_user_from_request_api_wrapper(app, request) async

API callable version of get_current_user_from_request for tbjs/admin panel

Source code in toolboxv2/mods/CloudM/UserAccountManager.py
150
151
152
153
154
155
156
157
158
159
160
161
@export(mod_name=Name, api=True, version=version, request_as_kwarg=True, row=False)  # row=False to return JSON
async def get_current_user_from_request_api_wrapper(app: App, request: RequestData):
    """ API callable version of get_current_user_from_request for tbjs/admin panel """
    user = await get_current_user_from_request(app, request)
    if not user:
        # Return error that tbjs can handle
        return Result.default_user_error(info="User not authenticated or found.", data=None, exec_code=401)
    user_dict = asdict(user)
    pub_user_data = {}
    for key in ['name','pub_key','email','creation_time','is_persona','level','log_level','settings']:
        pub_user_data[key] = user_dict.get(key, None)
    return Result.ok(data=pub_user_data)

UserDashboard

close_cli_session(app, request, cli_session_id) async

Close a specific CLI session

Source code in toolboxv2/mods/CloudM/UserDashboard.py
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
@export(mod_name=Name, version=version, api=True, request_as_kwarg=True)
async def close_cli_session(app: App, request: RequestData, cli_session_id: str):
    """Close a specific CLI session"""
    current_user = await get_current_user_from_request(app, request)
    if not current_user:
        return Result.default_user_error(info="User not authenticated.", exec_code=401)

    from .UserInstances import close_cli_session as close_cli_session_internal, UserInstances

    # Verify session belongs to current user
    if cli_session_id in UserInstances().cli_sessions:
        session_data = UserInstances().cli_sessions[cli_session_id]
        if session_data['uid'] != current_user.uid:
            return Result.default_user_error(info="Unauthorized to close this session.")

    result = close_cli_session_internal(cli_session_id)
    return Result.ok(data_info="CLI session closed", data={"message": result})
configure_jwt_settings(app=None, ttl_hours=24, allowed_mods=None)

Configure JWT settings for CLI sessions

Source code in toolboxv2/mods/CloudM/UserDashboard.py
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
@export(mod_name=Name, version=version, api=True)
def configure_jwt_settings(app: App = None, ttl_hours: int = 24, allowed_mods: list = None):
    """Configure JWT settings for CLI sessions"""
    if app is None:
        app = get_app("CloudM.configure_jwt_settings")

    if allowed_mods is None:
        allowed_mods = ["CloudM", "DB", "FileHandler"]

    jwt_config = {
        'ttl_hours': ttl_hours,
        'allowed_mods': allowed_mods,
        'secret_key': app.config_fh.get_file_handler("jwt_secret") or _generate_jwt_secret(app),
        'algorithm': 'HS256'
    }

    app.config_fh.add_to_save_file_handler("jwt_config", jwt_config)
    return Result.ok(data_info="JWT configuration updated", data=jwt_config)
generate_cli_jwt(app=None, username=None, custom_ttl=None)

Generate JWT token for CLI access

Source code in toolboxv2/mods/CloudM/UserDashboard.py
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
@export(mod_name=Name, version=version, api=True)
def generate_cli_jwt(app: App = None, username: str = None, custom_ttl: int = None):
    """Generate JWT token for CLI access"""
    if app is None:
        app = get_app("CloudM.generate_cli_jwt")

    if not username:
        return Result.default_user_error("Username required")

    jwt_config = app.config_fh.get_file_handler("jwt_config")
    if not jwt_config:
        # Use default configuration
        configure_jwt_settings(app)
        jwt_config = app.config_fh.get_file_handler("jwt_config")

    ttl_hours = custom_ttl or jwt_config.get('ttl_hours', 24)

    payload = {
        'username': username,
        'iat': int(time.time()),
        'exp': int(time.time()) + (ttl_hours * 3600),
        'type': 'cli_access',
        'allowed_mods': jwt_config.get('allowed_mods', [])
    }

    token = jwt.encode(
        payload,
        jwt_config['secret_key'],
        algorithm=jwt_config['algorithm']
    )

    return Result.ok(data_info="JWT generated", data={'token': token, 'expires_in': ttl_hours * 3600})
get_jwt_dashboard(app=None)

Get JWT management dashboard data

Source code in toolboxv2/mods/CloudM/UserDashboard.py
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
@export(mod_name=Name, version=version, api=True)
def get_jwt_dashboard(app: App = None):
    """Get JWT management dashboard data"""
    if app is None:
        app = get_app("CloudM.get_jwt_dashboard")

    jwt_config = app.config_fh.get_file_handler("jwt_config")

    dashboard_data = {
        'jwt_configured': jwt_config is not None,
        'current_config': jwt_config or {},
        'active_sessions': _get_active_sessions(app),
        'default_ttl': jwt_config.get('ttl_hours', 24) if jwt_config else 24
    }

    return Result.ok(data_info="Dashboard data", data=dashboard_data)
get_my_active_instances_with_cli(app, request) async

Get active instances including CLI sessions

Source code in toolboxv2/mods/CloudM/UserDashboard.py
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
@export(mod_name=Name, version=version, api=True, request_as_kwarg=True)
async def get_my_active_instances_with_cli(app: App, request: RequestData):
    """Get active instances including CLI sessions"""
    current_user = await get_current_user_from_request(app, request)
    if not current_user:
        return Result.default_user_error(info="User not authenticated.", exec_code=401)

    # Get enhanced instance data with CLI sessions
    from .UserInstances import get_user_instance_with_cli_sessions, get_user_cli_sessions

    instance_data_result = get_user_instance_with_cli_sessions(current_user.uid, hydrate=True)
    cli_sessions = get_user_cli_sessions(current_user.uid)

    active_instances_output = []
    if instance_data_result and isinstance(instance_data_result, dict):
        live_modules_info = []
        if instance_data_result.get("live"):
            for mod_name, spec_val in instance_data_result.get("live").items():
                live_modules_info.append({"name": mod_name, "spec": str(spec_val)})

        instance_summary = {
            "SiID": instance_data_result.get("SiID"),
            "VtID": instance_data_result.get("VtID"),
            "webSocketID": instance_data_result.get("webSocketID"),
            "live_modules": live_modules_info,
            "saved_modules": instance_data_result.get("save", {}).get("mods", []),
            "cli_sessions": cli_sessions,
            "active_cli_sessions": len([s for s in cli_sessions if s.get('status') == 'active'])
        }
        active_instances_output.append(instance_summary)

    return Result.ok(data_info="Active instances with CLI sessions retrieved", data=active_instances_output)
validate_cli_jwt(app=None, token=None)

Validate CLI JWT token

Source code in toolboxv2/mods/CloudM/UserDashboard.py
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
@export(mod_name=Name, version=version, api=True)
def validate_cli_jwt(app: App = None, token: str = None):
    """Validate CLI JWT token"""
    if app is None:
        app = get_app("CloudM.validate_cli_jwt")

    if not token:
        return Result.default_user_error("Token required")

    jwt_config = app.config_fh.get_file_handler("jwt_config")
    if not jwt_config:
        return Result.default_internal_error("JWT configuration not found")

    try:
        payload = jwt.decode(
            token,
            jwt_config['secret_key'],
            algorithms=[jwt_config['algorithm']]
        )

        # Check if token is for CLI access
        if payload.get('type') != 'cli_access':
            return Result.default_user_error("Invalid token type")

        return Result.ok(data_info="Token valid", data=payload)

    except jwt.ExpiredSignatureError:
        return Result.default_user_error("Token expired")
    except jwt.InvalidTokenError:
        return Result.default_user_error("Invalid token")

UserInstances

cleanup_expired_cli_sessions(max_age_hours=24)

Clean up expired CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
@e
def cleanup_expired_cli_sessions(max_age_hours: int = 24):
    """Clean up expired CLI sessions"""
    current_time = time.time()
    max_age_seconds = max_age_hours * 3600

    expired_sessions = []
    for session_id, session_data in list(UserInstances().cli_sessions.items()):
        if current_time - session_data['last_activity'] > max_age_seconds:
            expired_sessions.append(session_id)

    for session_id in expired_sessions:
        close_cli_session(session_id)

    logger.info(f"Cleaned up {len(expired_sessions)} expired CLI sessions")
    return f"Cleaned up {len(expired_sessions)} expired CLI sessions"
close_cli_session(cli_session_id)

Close a CLI session

Source code in toolboxv2/mods/CloudM/UserInstances.py
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
@e
def close_cli_session(cli_session_id: str):
    """Close a CLI session"""
    if cli_session_id not in UserInstances().cli_sessions:
        return "CLI session not found"

    session_data = UserInstances().cli_sessions[cli_session_id]
    session_data['status'] = 'closed'
    session_data['closed_at'] = time.time()

    # Remove from active sessions
    del UserInstances().cli_sessions[cli_session_id]

    # Update persistent storage to mark as closed
    app.run_any('DB', 'set',
                query=f"CLI::Session::{session_data['uid']}::{cli_session_id}",
                data=json.dumps(session_data))

    logger.info(f"CLI session {cli_session_id} closed")
    return "CLI session closed successfully"
get_all_active_cli_sessions()

Get all active CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py
321
322
323
324
@e
def get_all_active_cli_sessions():
    """Get all active CLI sessions"""
    return list(UserInstances().cli_sessions.values())
get_instance_overview(si_id=None)

Get comprehensive overview of user instances and CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
@e
def get_instance_overview(si_id: str = None):
    """Get comprehensive overview of user instances and CLI sessions"""
    overview = {
        'web_instances': {},
        'cli_sessions': {},
        'total_active_web': 0,
        'total_active_cli': 0
    }

    # Web instances
    if si_id:
        if si_id in UserInstances().live_user_instances:
            overview['web_instances'][si_id] = UserInstances().live_user_instances[si_id]
            overview['total_active_web'] = 1
    else:
        overview['web_instances'] = UserInstances().live_user_instances.copy()
        overview['total_active_web'] = len(UserInstances().live_user_instances)

    # CLI sessions
    overview['cli_sessions'] = UserInstances().cli_sessions.copy()
    overview['total_active_cli'] = len(UserInstances().cli_sessions)

    return overview
get_user_cli_sessions(uid)

Get all CLI sessions for a user

Source code in toolboxv2/mods/CloudM/UserInstances.py
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
@e
def get_user_cli_sessions(uid: str):
    """Get all CLI sessions for a user"""
    if uid is None:
        return []

    # Get active sessions from memory
    active_sessions = []
    for session_id, session_data in UserInstances().cli_sessions.items():
        if session_data['uid'] == uid:
            active_sessions.append(session_data)

    # Also check persistent storage for recent sessions
    try:
        # This would need a query pattern to get all CLI sessions for a user
        # For now, return active sessions
        pass
    except Exception as e:
        logger.warning(f"Error fetching persistent CLI sessions: {e}")

    return active_sessions
get_user_instance_with_cli_sessions(uid, hydrate=True)

Enhanced get_user_instance that includes CLI sessions

Source code in toolboxv2/mods/CloudM/UserInstances.py
345
346
347
348
349
350
351
352
353
354
355
356
357
@e
def get_user_instance_with_cli_sessions(uid: str, hydrate: bool = True):
    """Enhanced get_user_instance that includes CLI sessions"""
    # Get regular user instance
    instance = get_user_instance(uid, hydrate)

    if instance:
        # Add CLI sessions information
        cli_sessions = get_user_cli_sessions(uid)
        instance['cli_sessions'] = cli_sessions
        instance['active_cli_sessions'] = len([s for s in cli_sessions if s['status'] == 'active'])

    return instance
register_cli_session(uid, jwt_token, session_info=None)

Register a new CLI session

Source code in toolboxv2/mods/CloudM/UserInstances.py
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
@e
def register_cli_session(uid: str, jwt_token: str, session_info: dict = None):
    """Register a new CLI session"""
    if uid is None:
        return Result.default_user_error("UID required")

    cli_session_id = UserInstances.get_cli_session_id(uid).get()

    session_data = {
        'uid': uid,
        'cli_session_id': cli_session_id,
        'jwt_token': jwt_token,
        'created_at': time.time(),
        'last_activity': time.time(),
        'status': 'active',
        'session_info': session_info or {}
    }

    UserInstances().cli_sessions[cli_session_id] = session_data

    # Save to persistent storage
    app.run_any('DB', 'set',
                query=f"CLI::Session::{uid}::{cli_session_id}",
                data=json.dumps(session_data))

    logger.info(f"CLI session registered for user {uid}")
    return Result.ok("CLI session registered", data=session_data)
update_cli_session_activity(cli_session_id)

Update last activity timestamp for CLI session

Source code in toolboxv2/mods/CloudM/UserInstances.py
261
262
263
264
265
266
267
268
269
270
271
272
273
@e
def update_cli_session_activity(cli_session_id: str):
    """Update last activity timestamp for CLI session"""
    if cli_session_id in UserInstances().cli_sessions:
        UserInstances().cli_sessions[cli_session_id]['last_activity'] = time.time()
        session_data = UserInstances().cli_sessions[cli_session_id]

        # Update persistent storage
        app.run_any('DB', 'set',
                    query=f"CLI::Session::{session_data['uid']}::{cli_session_id}",
                    data=json.dumps(session_data))
        return True
    return False

email_services

send_email_verification_email(app, user_email, username, verification_url)

Sends an email verification link to the user.

Source code in toolboxv2/mods/CloudM/email_services.py
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
@s_export
def send_email_verification_email(app: App, user_email: str, username: str, verification_url: str):
    """Sends an email verification link to the user."""
    sender = EmailSender(app)
    subject = f"Verify Your Email for {APP_NAME}"
    preview_text = f"Almost there, {username}! Just one more step to activate your account."

    content_html = f"""
        <h2>Hi {username},</h2>
        <p>Thanks for signing up for {APP_NAME}! To complete your registration, please verify your email address by clicking the button below.</p>
        <a href="{verification_url}" class="button">Verify Email Address</a>
        <p>If you didn't create an account with {APP_NAME}, you can safely ignore this email.</p>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{verification_url}</span></p>
        <p>Sincerely,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text)

Sends a magic link email for login.

Source code in toolboxv2/mods/CloudM/email_services.py
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
@s_export
def send_magic_link_email(app: App, user_email: str, magic_link_url: str, username: str = None):
    """Sends a magic link email for login."""
    sender = EmailSender(app)
    greeting_name = f", {username}" if username else ""
    subject = f"Your Magic Login Link for {APP_NAME}"
    preview_text = "Securely access your account with this one-time link."

    content_html = f"""
        <h2>Hello{greeting_name}!</h2>
        <p>You requested a magic link to sign in to your {APP_NAME} account.</p>
        <p>Click the button below to log in. This link is temporary and will expire shortly.</p>
        <a href="{magic_link_url}" class="button">Log In Securely</a>
        <p> Invitation key: {magic_link_url.split('?key=')[1].split('&name=')[0].replace('%23', '#')}</p>
        <p>If you did not request this link, please ignore this email. Your account is safe.</p>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{magic_link_url}</span></p>
        <p>Thanks,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text)
send_signup_invitation_email(app, invited_user_email, invited_username, inviter_username=None)

Generates an invitation link and sends it via email.

Source code in toolboxv2/mods/CloudM/email_services.py
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
@s_export
def send_signup_invitation_email(app: App, invited_user_email: str, invited_username: str,
                                 inviter_username: str = None):
    """Generates an invitation link and sends it via email."""
    sender = EmailSender(app)

    # Generate invitation code as specified in the prompt
    # This uses the Code class, assuming TB_R_KEY is set in the environment
    invitation_code = Code.one_way_hash(invited_username, "00#", os.getenv("TB_R_KEY", "pepper123"))[:12] + str(
        uuid.uuid4())[:6]

    # Construct the signup link URL (adjust your frontend signup path as needed)
    signup_link_url = f"{APP_BASE_URL}/web/assets/signup.html?invitation={quote(invitation_code)}&email={quote(invited_user_email)}&username={quote(invited_username)}"

    subject = f"You're Invited to Join {APP_NAME}!"
    preview_text = f"{inviter_username or 'A friend'} has invited you to {APP_NAME}!"
    inviter_line = f"<p>{inviter_username} has invited you to join.</p>" if inviter_username else "<p>You've been invited to join.</p>"

    content_html = f"""
        <h2>Hello {invited_username},</h2>
        {inviter_line}
        <p>{APP_NAME} is an exciting platform, and we'd love for you to be a part of it!</p>
        <p>Click the button below to accept the invitation and create your account:</p>
        <a href="{signup_link_url}" class="button">Accept Invitation & Sign Up</a>
        <p>This invitation is unique to you : {invitation_code}</p>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{signup_link_url}</span></p>
        <p>We look forward to seeing you there!<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(invited_user_email, subject, content_html, preview_text)
send_waiting_list_confirmation_email(app, user_email)

Sends a confirmation email for joining the waiting list.

Source code in toolboxv2/mods/CloudM/email_services.py
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
@s_export
def send_waiting_list_confirmation_email(app: App, user_email: str):
    """Sends a confirmation email for joining the waiting list."""
    sender = EmailSender(app)
    subject = f"You're on the Waiting List for {APP_NAME}!"
    preview_text = "Thanks for your interest! We'll keep you updated."

    content_html = f"""
        <h2>You're In!</h2>
        <p>Thank you for joining the waiting list for {APP_NAME}. We're working hard to get things ready and appreciate your interest.</p>
        <p>We'll notify you as soon as we have updates or when access becomes available.</p>
        <p>In the meantime, you can follow our progress or learn more at <a href="{APP_BASE_URL}" class="link-in-text">{APP_BASE_URL}</a>.</p>
        <p>Stay tuned,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text,
                                  recipient_email_for_unsubscribe=user_email, show_unsubscribe_link=True)
send_welcome_email(app, user_email, username, welcome_action_url=None)

Sends a welcome email to a new user.

Source code in toolboxv2/mods/CloudM/email_services.py
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
@s_export  # Changed to native, api=False as it's a backend function
def send_welcome_email(app: App, user_email: str, username: str, welcome_action_url: str = None):
    """Sends a welcome email to a new user."""
    sender = EmailSender(app)
    subject = f"Welcome to {APP_NAME}, {username}!"
    preview_text = f"We're thrilled to have you, {username}!"
    action_url = welcome_action_url or f"{APP_BASE_URL}/dashboard"  # Default to dashboard

    content_html = f"""
        <h2>Welcome Aboard, {username}!</h2>
        <p>Thank you for signing up for {APP_NAME}. We're excited to have you join our community!</p>
        <p>Here are a few things you might want to do next:</p>
        <ul>
            <li>Explore your new account features.</li>
            <li>Customize your profile.</li>
        </ul>
        <p>Click the button below to get started:</p>
        <a href="{action_url}" class="button">Go to Your Dashboard</a>
        <p>If the button doesn't work, copy and paste this link into your browser:<br><span class="link-in-text">{action_url}</span></p>
        <p>Best regards,<br>The {APP_NAME} Team</p>
    """
    return sender.send_html_email(user_email, subject, content_html, preview_text,
                                  recipient_email_for_unsubscribe=user_email, show_unsubscribe_link=True)

extras

cli_logout(app=None) async

CLI logout - imports from LogInSystem

Source code in toolboxv2/mods/CloudM/extras.py
369
370
371
372
373
@export(mod_name=Name, version=version, state=False)
async def cli_logout(app: App = None):
    """CLI logout - imports from LogInSystem"""
    from .LogInSystem import cli_logout as _cli_logout
    return await _cli_logout(app)
cli_web_login(app=None, force_remote=False, force_local=False) async

Enhanced CLI web login - imports from LogInSystem

Source code in toolboxv2/mods/CloudM/extras.py
363
364
365
366
367
@export(mod_name=Name, version=version, state=False)
async def cli_web_login(app: App = None, force_remote: bool = False, force_local: bool = False):
    """Enhanced CLI web login - imports from LogInSystem"""
    from .LogInSystem import cli_web_login as _cli_web_login
    return await _cli_web_login(app, force_remote, force_local)

mini

check_multiple_processes(pids)

Checks the status of multiple processes in a single system call. Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).

Source code in toolboxv2/mods/CloudM/mini.py
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
def check_multiple_processes(pids: list[int]) -> dict[int, str]:
    """
    Checks the status of multiple processes in a single system call.
    Returns a dictionary mapping PIDs to their status (GREEN_CIRCLE, RED_CIRCLE, or YELLOW_CIRCLE).
    """
    if not pids:
        return {}

    pid_status = {}

    if os.name == 'nt':  # Windows
        try:
            # Windows tasklist requires separate /FI for each filter
            command = 'tasklist'

            # Add encoding handling for Windows
            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='cp850'  # Use cp850 for Windows console output
            )
            # Create a set of running PIDs from the output
            running_pids = set()
            for line in result.stdout.lower().split('\n'):
                for pid in pids:
                    if str(pid) in line:
                        running_pids.add(pid)
            # Assign status based on whether PID was found in output
            for pid in pids:
                if pid in running_pids:
                    pid_status[pid] = GREEN_CIRCLE
                else:
                    pid_status[pid] = RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            # Mark all as YELLOW_CIRCLE if there's an error running the command
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE
        except UnicodeDecodeError as e:
            print(f"UnicodeDecodeError: {e}")  # For debugging
            # Try alternate encoding if cp850 fails
            try:
                result = subprocess.run(
                    command,
                    capture_output=True,
                    text=True,
                    shell=True,
                    encoding='utf-8'
                )
                running_pids = set()
                for line in result.stdout.lower().split('\n'):
                    for pid in pids:
                        if str(pid) in line:
                            running_pids.add(pid)

                for pid in pids:
                    pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE
            except Exception as e:
                print(f"Failed with alternate encoding: {e}")  # For debugging
                for pid in pids:
                    pid_status[pid] = YELLOW_CIRCLE

    else:  # Unix/Linux/Mac
        try:
            pids_str = ','.join(str(pid) for pid in pids)
            command = f'ps -p {pids_str} -o pid='

            result = subprocess.run(
                command,
                capture_output=True,
                text=True,
                shell=True,
                encoding='utf-8'
            )
            running_pids = set(int(pid) for pid in result.stdout.strip().split())

            for pid in pids:
                pid_status[pid] = GREEN_CIRCLE if pid in running_pids else RED_CIRCLE

        except subprocess.SubprocessError as e:
            print(f"SubprocessError: {e}")  # For debugging
            for pid in pids:
                pid_status[pid] = YELLOW_CIRCLE

    return pid_status
get_service_pids(info_dir)

Extracts service names and PIDs from pid files.

Source code in toolboxv2/mods/CloudM/mini.py
14
15
16
17
18
19
20
21
22
23
24
25
26
27
def get_service_pids(info_dir):
    """Extracts service names and PIDs from pid files."""
    services = {}
    pid_files = [f for f in os.listdir(info_dir) if re.match(r'(.+)-(.+)\.pid', f)]
    for pid_file in pid_files:
        match = re.match(r'(.+)-(.+)\.pid', pid_file)
        if match:
            services_type, service_name = match.groups()
            # Read the PID from the file
            with open(os.path.join(info_dir, pid_file)) as file:
                pid = file.read().strip()
                # Store the PID using a formatted key
                services[f"{service_name} - {services_type}"] = int(pid)
    return services
get_service_status(dir)

Displays the status of all services.

Source code in toolboxv2/mods/CloudM/mini.py
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
def get_service_status(dir: str) -> str:
    """Displays the status of all services."""
    if time.time()-services_data_sto_last_update_time[0] > 30:
        services = get_service_pids(dir)
        services_data_sto[0] = services
        services_data_sto_last_update_time[0] = time.time()
    else:
        services = services_data_sto[0]
    if not services:
        return "No services found"

    # Get status for all PIDs in a single call
    pid_statuses = check_multiple_processes(list(services.values()))

    # Build the status string
    res_s = "Service(s):" + ("\n" if len(services) > 1 else ' ')
    for service_name, pid in services.items():
        status = pid_statuses.get(pid, YELLOW_CIRCLE)
        res_s += f"{status} {service_name} (PID: {pid})\n"
    services_data_display[0] = res_s.strip()
    return res_s.rstrip()

module

hash_password(password)

Hash a password for storing.

Source code in toolboxv2/mods/CloudM/module.py
109
110
111
112
113
114
115
def hash_password(password):
    """Hash a password for storing."""
    salt = hashlib.sha256(os.urandom(60)).hexdigest().encode('ascii')
    pwdhash = hashlib.pbkdf2_hmac('sha512', password.encode('utf-8'), salt,
                                  100000)
    pwdhash = binascii.hexlify(pwdhash)
    return (salt + pwdhash).decode('ascii')
verify_password(stored_password, provided_password)

Verify a stored password against one provided by user

Source code in toolboxv2/mods/CloudM/module.py
119
120
121
122
123
124
125
126
def verify_password(stored_password, provided_password):
    """Verify a stored password against one provided by user"""
    salt = stored_password[:64]
    stored_password = stored_password[64:]
    pwdhash = hashlib.pbkdf2_hmac('sha512', provided_password.encode('utf-8'),
                                  salt.encode('ascii'), 100000)
    pwdhash = binascii.hexlify(pwdhash).decode('ascii')
    return pwdhash == stored_password

CodeVerification

VerificationSystem

Source code in toolboxv2/mods/CodeVerification.py
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
class VerificationSystem:
    def __init__(self, tools_db, scope="main"):
        """
        Initialize VerificationSystem with DB Tools integration

        Args:
            tools_db (Tools): Database tools from toolboxv2.mods.DB
            scope (str, optional): Scope for templates and codes. Defaults to "main".
        """
        self.tools_db = tools_db
        self.scope = scope
        self.tidmp = {}
        self._ensure_scope_templates()

    def get(self):
        return self

    def reset_scope_templates(self):
        """
        Ensure a templates dictionary exists for the current scope in the database
        """
        templates_key = f"verification_templates_{self.scope}"

        self.tools_db.set(templates_key, json.dumps({}))

    def _ensure_scope_templates(self):
        """
        Ensure a templates dictionary exists for the current scope in the database
        """
        templates_key = f"verification_templates_{self.scope}"

        # Check if templates exist for this scope
        templates_exist = self.tools_db.if_exist(templates_key)

        if templates_exist.is_error() and not templates_exist.is_data():
            # Initialize empty templates dictionary if not exists
            self.tools_db.set(templates_key, json.dumps({}))
        else:
            allt = self.get_all_templates()

            for k, v in allt.items():
                if 'name' not in v:
                    continue
                self.tidmp[v['name']] = k

    def add_config_template(self, template: ConfigTemplate) -> str:
        """
        Add a new configuration template to the database

        Args:
            template (ConfigTemplate): The configuration template

        Returns:
            str: Unique identifier of the template
        """
        # Ensure template has the current scope
        template.scope = self.scope

        # Generate a unique template ID
        template_id = secrets.token_urlsafe(8)

        # Get existing templates for this scope
        templates = self.get_all_templates()

        # Add new template
        self.tidmp[template.name] = template_id
        templates[template_id] = asdict(template)

        # Save updated templates back to database
        templates_key = f"verification_templates_{self.scope}"
        save_result = self.tools_db.set(templates_key, json.dumps(templates))

        if save_result.is_error():
            raise ValueError("Could not save template")

        return template_id

    def get_all_templates(self):
        templates_key = f"verification_templates_{self.scope}"
        templates_result = self.tools_db.get(templates_key)

        if not templates_result.is_error() and templates_result.is_data():
            try:
                templates_result.result.data = json.loads(templates_result.get())
            except Exception as e:
                templates_result.print()
                print(f"Errro loding template data curupted : {str(e)}")
                templates_result.result.data = {}
        else:
            templates_result.result.data = {}
        if not isinstance(templates_result, dict):
            templates_result = templates_result.result.data
        return templates_result

    def generate_code(self, template_id: str) -> str:
        """
        Generate a code based on the configuration template

        Args:
            template_id (str): ID of the configuration template

        Returns:
            str: Generated verification code
        """
        # Get templates for this scope
        templates = self.get_all_templates()
        print(templates, self.tidmp, template_id)
        if template_id not in templates:
            template_id = self.tidmp.get(template_id, template_id)
        if template_id not in templates:
            raise ValueError("Invalid configuration template")

        template_dict = templates[template_id]
        ConfigTemplate(**template_dict)

        # Generate a random code with max 16 characters
        code = secrets.token_urlsafe(10)[:16]

        # Prepare code information
        code_info = {
            'template_id': template_id,
            'created_at': time.time(),
            'uses_count': 0,
            'scope': self.scope
        }

        # Store code information in database
        codes_key = f"verification_codes_{self.scope}"
        existing_codes_result = self.tools_db.get(codes_key)

        existing_codes = {}
        if not existing_codes_result.is_error() and existing_codes_result.is_data():
            d = existing_codes_result.get()
            if isinstance(d, list):
                d = d[0]
            existing_codes = json.loads(d)

        existing_codes[code] = code_info

        save_result = self.tools_db.set(codes_key, json.dumps(existing_codes))

        if save_result.is_error():
            raise ValueError("Could not save generated code")

        return code

    def validate_code(self, code: str) -> dict[str, Any] | None:
        """
        Validate a code and return template information

        Args:
            code (str): Code to validate

        Returns:
            Optional[Dict[str, Any]]: Template information for valid code, else None
        """
        # Get codes for this scope
        codes_key = f"verification_codes_{self.scope}"
        codes_result = self.tools_db.get(codes_key)

        if codes_result.is_error() or not codes_result.is_data():
            return None

        d = codes_result.get()
        if isinstance(d, list):
            d = d[0]
        existing_codes = json.loads(d)

        if code not in existing_codes:
            return None

        code_info = existing_codes[code]

        # Check if code is from the same scope
        if code_info.get('scope') != self.scope:
            return None

        # Get templates for this scope
        templates = self.get_all_templates()
        template_id = code_info['template_id']

        if template_id not in templates:
            return templates

        template_dict = templates[template_id]
        template = ConfigTemplate(**template_dict)

        # Check usage count
        if code_info['uses_count'] >= template.max_uses:
            del existing_codes[code]
            self.tools_db.set(codes_key, json.dumps(existing_codes))
            return None

        # Check time validity for timed codes
        if template.usage_type == 'timed':
            current_time = time.time()
            if template.valid_duration and (current_time - code_info['created_at']) > template.valid_duration:
                del existing_codes[code]
                self.tools_db.set(codes_key, json.dumps(existing_codes))
                return None

        # Update uses count
        existing_codes[code]['uses_count'] += 1
        uses_count = existing_codes[code].get('uses_count', 1)
        # Remove code if it's a one-time use
        if template.usage_type == 'one_time':
            del existing_codes[code]

        # Save updated codes
        self.tools_db.set(codes_key, json.dumps(existing_codes))

        return {
            'template_name': template.name,
            'usage_type': template.usage_type,
            'uses_count': uses_count
        }
__init__(tools_db, scope='main')

Initialize VerificationSystem with DB Tools integration

Parameters:

Name Type Description Default
tools_db Tools

Database tools from toolboxv2.mods.DB

required
scope str

Scope for templates and codes. Defaults to "main".

'main'
Source code in toolboxv2/mods/CodeVerification.py
27
28
29
30
31
32
33
34
35
36
37
38
def __init__(self, tools_db, scope="main"):
    """
    Initialize VerificationSystem with DB Tools integration

    Args:
        tools_db (Tools): Database tools from toolboxv2.mods.DB
        scope (str, optional): Scope for templates and codes. Defaults to "main".
    """
    self.tools_db = tools_db
    self.scope = scope
    self.tidmp = {}
    self._ensure_scope_templates()
add_config_template(template)

Add a new configuration template to the database

Parameters:

Name Type Description Default
template ConfigTemplate

The configuration template

required

Returns:

Name Type Description
str str

Unique identifier of the template

Source code in toolboxv2/mods/CodeVerification.py
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
def add_config_template(self, template: ConfigTemplate) -> str:
    """
    Add a new configuration template to the database

    Args:
        template (ConfigTemplate): The configuration template

    Returns:
        str: Unique identifier of the template
    """
    # Ensure template has the current scope
    template.scope = self.scope

    # Generate a unique template ID
    template_id = secrets.token_urlsafe(8)

    # Get existing templates for this scope
    templates = self.get_all_templates()

    # Add new template
    self.tidmp[template.name] = template_id
    templates[template_id] = asdict(template)

    # Save updated templates back to database
    templates_key = f"verification_templates_{self.scope}"
    save_result = self.tools_db.set(templates_key, json.dumps(templates))

    if save_result.is_error():
        raise ValueError("Could not save template")

    return template_id
generate_code(template_id)

Generate a code based on the configuration template

Parameters:

Name Type Description Default
template_id str

ID of the configuration template

required

Returns:

Name Type Description
str str

Generated verification code

Source code in toolboxv2/mods/CodeVerification.py
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
def generate_code(self, template_id: str) -> str:
    """
    Generate a code based on the configuration template

    Args:
        template_id (str): ID of the configuration template

    Returns:
        str: Generated verification code
    """
    # Get templates for this scope
    templates = self.get_all_templates()
    print(templates, self.tidmp, template_id)
    if template_id not in templates:
        template_id = self.tidmp.get(template_id, template_id)
    if template_id not in templates:
        raise ValueError("Invalid configuration template")

    template_dict = templates[template_id]
    ConfigTemplate(**template_dict)

    # Generate a random code with max 16 characters
    code = secrets.token_urlsafe(10)[:16]

    # Prepare code information
    code_info = {
        'template_id': template_id,
        'created_at': time.time(),
        'uses_count': 0,
        'scope': self.scope
    }

    # Store code information in database
    codes_key = f"verification_codes_{self.scope}"
    existing_codes_result = self.tools_db.get(codes_key)

    existing_codes = {}
    if not existing_codes_result.is_error() and existing_codes_result.is_data():
        d = existing_codes_result.get()
        if isinstance(d, list):
            d = d[0]
        existing_codes = json.loads(d)

    existing_codes[code] = code_info

    save_result = self.tools_db.set(codes_key, json.dumps(existing_codes))

    if save_result.is_error():
        raise ValueError("Could not save generated code")

    return code
reset_scope_templates()

Ensure a templates dictionary exists for the current scope in the database

Source code in toolboxv2/mods/CodeVerification.py
43
44
45
46
47
48
49
def reset_scope_templates(self):
    """
    Ensure a templates dictionary exists for the current scope in the database
    """
    templates_key = f"verification_templates_{self.scope}"

    self.tools_db.set(templates_key, json.dumps({}))
validate_code(code)

Validate a code and return template information

Parameters:

Name Type Description Default
code str

Code to validate

required

Returns:

Type Description
dict[str, Any] | None

Optional[Dict[str, Any]]: Template information for valid code, else None

Source code in toolboxv2/mods/CodeVerification.py
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
def validate_code(self, code: str) -> dict[str, Any] | None:
    """
    Validate a code and return template information

    Args:
        code (str): Code to validate

    Returns:
        Optional[Dict[str, Any]]: Template information for valid code, else None
    """
    # Get codes for this scope
    codes_key = f"verification_codes_{self.scope}"
    codes_result = self.tools_db.get(codes_key)

    if codes_result.is_error() or not codes_result.is_data():
        return None

    d = codes_result.get()
    if isinstance(d, list):
        d = d[0]
    existing_codes = json.loads(d)

    if code not in existing_codes:
        return None

    code_info = existing_codes[code]

    # Check if code is from the same scope
    if code_info.get('scope') != self.scope:
        return None

    # Get templates for this scope
    templates = self.get_all_templates()
    template_id = code_info['template_id']

    if template_id not in templates:
        return templates

    template_dict = templates[template_id]
    template = ConfigTemplate(**template_dict)

    # Check usage count
    if code_info['uses_count'] >= template.max_uses:
        del existing_codes[code]
        self.tools_db.set(codes_key, json.dumps(existing_codes))
        return None

    # Check time validity for timed codes
    if template.usage_type == 'timed':
        current_time = time.time()
        if template.valid_duration and (current_time - code_info['created_at']) > template.valid_duration:
            del existing_codes[code]
            self.tools_db.set(codes_key, json.dumps(existing_codes))
            return None

    # Update uses count
    existing_codes[code]['uses_count'] += 1
    uses_count = existing_codes[code].get('uses_count', 1)
    # Remove code if it's a one-time use
    if template.usage_type == 'one_time':
        del existing_codes[code]

    # Save updated codes
    self.tools_db.set(codes_key, json.dumps(existing_codes))

    return {
        'template_name': template.name,
        'usage_type': template.usage_type,
        'uses_count': uses_count
    }

DB

blob_instance

BlobDB

A persistent, encrypted dictionary-like database that uses the BlobStorage system as its backend, making it networked and fault-tolerant.

This implementation uses multiple BlobFiles instead of a single virtual file. Keys are structured like USER::XYZ::: or MANAGER::SPACE::DATA::XYZ and are converted to file paths by replacing :: with /.

Source code in toolboxv2/mods/DB/blob_instance.py
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
class BlobDB:
    """
    A persistent, encrypted dictionary-like database that uses the BlobStorage
    system as its backend, making it networked and fault-tolerant.

    This implementation uses multiple BlobFiles instead of a single virtual file.
    Keys are structured like USER::XYZ::: or MANAGER::SPACE::DATA::XYZ and are
    converted to file paths by replacing :: with /.
    """
    auth_type = AuthenticationTypes.location

    def __init__(self):
        self.data: dict = {}  # In-memory cache of all data
        self.key: str | None = None
        self.db_path: str | None = None  # Base path for the database
        self.storage_client: BlobStorage | None = None


    def _key_to_blob_path(self, key: str) -> str:
        """
        Converts a database key to a blob file path.
        Replaces :: with / to create a hierarchical file structure.

        Examples:
            USER::XYZ::: -> db_path/USER/XYZ.json
            MANAGER::SPACE::DATA::XYZ -> db_path/MANAGER/SPACE/DATA/XYZ.json
        """
        # Replace :: with / and remove trailing/leading colons and slashes
        path_parts = key.replace('::', '/').strip('/').strip(':').strip('/')
        # Combine with base db_path
        return f"{self.db_path}/{path_parts}.json"

    def _load_blob_file(self, blob_path: str) -> dict:
        """
        Loads data from a specific blob file.
        Returns empty dict if file doesn't exist.
        """
        try:
            db_file = BlobFile(blob_path, mode='r', storage=self.storage_client, key=self.key)
            if not db_file.exists():
                return {}
            with db_file as f:
                data = f.read_json()
                return data if data else {}
        except Exception as e:
            print(f"Warning: Could not load blob file '{blob_path}'. Error: {e}")
            return {}

    def _save_blob_file(self, blob_path: str, data: dict) -> bool:
        """
        Saves data to a specific blob file.
        Returns True on success, False on failure.
        """
        try:
            # Ensure the blob exists first
            db_file = BlobFile(blob_path, mode='r', storage=self.storage_client, key=self.key)
            if not db_file.exists():
                db_file.create()

            with BlobFile(blob_path, mode='w', storage=self.storage_client, key=self.key) as f:
                f.write_json(data)
            return True
        except Exception as e:
            print(f"Error: Could not save blob file '{blob_path}'. Error: {e}")
            return False

    def initialize(self, db_path: str, key: str, storage_client: BlobStorage) -> Result:
        """
        Initializes the database from a location within the blob storage.

        Args:
            db_path (str): The base path within the blob storage,
                           e.g., "my_database_blob".
            key (str): The encryption key for the database content.
            storage_client (BlobStorage): An initialized BlobStorage client instance.

        Returns:
            Result: An OK result if successful.
        """
        self.db_path = db_path.rstrip('/')
        self.key = key
        self.storage_client = storage_client

        print(f"Initializing BlobDB with base path: '{self.db_path}'...")

        # Initialize with empty data - data will be loaded on-demand
        self.data = {}

        print("Successfully initialized database with multi-file storage.")
        return Result.ok().set_origin("Blob Dict DB")

    def exit(self) -> Result:
        """
        Saves the current state of the database back to the blob storage.
        Each key is saved to its own blob file based on the key structure.
        """
        print("BLOB DB on exit ", not all([self.key, self.db_path, self.storage_client]))
        if not all([self.key, self.db_path, self.storage_client]):
            return Result.default_internal_error(
                info="Database not initialized. Cannot exit."
            ).set_origin("Blob Dict DB")

        print(f"Saving database to blob storage at base path: '{self.db_path}'...")

        errors = []
        saved_count = 0

        try:
            # Group keys by their blob file path
            blob_files_data = {}
            for key, value in self.data.items():
                blob_path = self._key_to_blob_path(key)
                if blob_path not in blob_files_data:
                    blob_files_data[blob_path] = {}
                blob_files_data[blob_path][key] = value

            # Save each blob file
            for blob_path, data in blob_files_data.items():
                if self._save_blob_file(blob_path, data):
                    saved_count += 1
                else:
                    errors.append(f"Failed to save {blob_path}")

            if errors:
                return Result.custom_error(
                    data=errors,
                    info=f"Saved {saved_count} blob files, but {len(errors)} failed: {errors}"
                ).set_origin("Blob Dict DB")

            print(f"Success: Database saved to {saved_count} blob file(s).")
            return Result.ok().set_origin("Blob Dict DB")

        except Exception as e:
            return Result.custom_error(
                data=e,
                info=f"Error saving database to blob storage: {e}"
            ).set_origin("Blob Dict DB")

    # --- Data Manipulation Methods ---
    # These methods now handle loading data on-demand from multiple blob files

    def _ensure_key_loaded(self, key: str):
        """
        Ensures that data for a specific key is loaded into memory.
        If the key contains wildcards, loads all matching blob files.
        """
        if key.endswith('*'):
            # For wildcard searches, we need to scan and load relevant files
            # This is a simplified approach - in production you might want to maintain an index
            prefix = key.replace('*', '')
            # For now, we'll rely on the in-memory cache
            # A more sophisticated approach would scan blob storage
            return

        # Check if key is already in memory
        if key in self.data:
            return

        # Load the blob file for this key
        blob_path = self._key_to_blob_path(key)
        blob_data = self._load_blob_file(blob_path)

        # Merge loaded data into memory
        self.data.update(blob_data)

    def get(self, key: str) -> Result:
        data = []

        if key == 'all':
            # Load all data - this is expensive, consider pagination in production
            data_info = "Returning all data available"
            data = list(self.data.items())
        elif key == "all-k":
            data_info = "Returning all keys"
            data = list(self.data.keys())
        else:
            # Ensure the key or matching keys are loaded
            self._ensure_key_loaded(key)
            data_info = f"Returning values for keys starting with '{key.replace('*', '')}'"
            data = [self.data[k] for k in self.scan_iter(key)]

        if not data:
            return Result.default_internal_error(info=f"No data found for key '{key}'").set_origin("Blob Dict DB")

        return Result.ok(data=data, data_info=data_info).set_origin("Blob Dict DB")

    def set(self, key: str, value) -> Result:
        if not isinstance(key, str) or not key:
            return Result.default_user_error(info="Key must be a non-empty string.").set_origin("Blob Dict DB")

        # Ensure existing data for this key is loaded first
        self._ensure_key_loaded(key)

        self.data[key] = value
        return Result.ok().set_origin("Blob Dict DB")

    def scan_iter(self, search: str = ''):
        if not self.data:
            return []
        prefix = search.replace('*', '')
        return [key for key in self.data if key.startswith(prefix)]

    def append_on_set(self, key: str, value: list) -> Result:
        # Ensure existing data for this key is loaded first
        self._ensure_key_loaded(key)

        if key not in self.data:
            self.data[key] = []

        if not isinstance(self.data[key], list):
            return Result.default_user_error(info=f"Existing value for key '{key}' is not a list.").set_origin(
                "Blob Dict DB")

        # Use a set for efficient checking to avoid duplicates
        existing_set = set(self.data[key])
        new_items = [item for item in value if item not in existing_set]
        self.data[key].extend(new_items)
        return Result.ok().set_origin("Blob Dict DB")

    def if_exist(self, key: str) -> int:
        # Ensure data is loaded for existence check
        self._ensure_key_loaded(key)

        if key.endswith('*'):
            return len(self.scan_iter(key))
        return 1 if key in self.data else 0

    def delete(self, key: str, matching: bool = False) -> Result:
        # Ensure data is loaded before deletion
        self._ensure_key_loaded(key)

        keys_to_delete = []
        if matching:
            keys_to_delete = self.scan_iter(key)
        elif key in self.data:
            keys_to_delete.append(key)

        if not keys_to_delete:
            return Result.default_internal_error(info=f"No keys found to delete for pattern '{key}'").set_origin(
                "Blob Dict DB")

        deleted_items = {k: self.data.pop(k) for k in keys_to_delete}
        return Result.ok(
            data=list(deleted_items.items()),
            data_info=f"Successfully removed {len(deleted_items)} item(s)."
        ).set_origin("Blob Dict DB")
exit()

Saves the current state of the database back to the blob storage. Each key is saved to its own blob file based on the key structure.

Source code in toolboxv2/mods/DB/blob_instance.py
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
def exit(self) -> Result:
    """
    Saves the current state of the database back to the blob storage.
    Each key is saved to its own blob file based on the key structure.
    """
    print("BLOB DB on exit ", not all([self.key, self.db_path, self.storage_client]))
    if not all([self.key, self.db_path, self.storage_client]):
        return Result.default_internal_error(
            info="Database not initialized. Cannot exit."
        ).set_origin("Blob Dict DB")

    print(f"Saving database to blob storage at base path: '{self.db_path}'...")

    errors = []
    saved_count = 0

    try:
        # Group keys by their blob file path
        blob_files_data = {}
        for key, value in self.data.items():
            blob_path = self._key_to_blob_path(key)
            if blob_path not in blob_files_data:
                blob_files_data[blob_path] = {}
            blob_files_data[blob_path][key] = value

        # Save each blob file
        for blob_path, data in blob_files_data.items():
            if self._save_blob_file(blob_path, data):
                saved_count += 1
            else:
                errors.append(f"Failed to save {blob_path}")

        if errors:
            return Result.custom_error(
                data=errors,
                info=f"Saved {saved_count} blob files, but {len(errors)} failed: {errors}"
            ).set_origin("Blob Dict DB")

        print(f"Success: Database saved to {saved_count} blob file(s).")
        return Result.ok().set_origin("Blob Dict DB")

    except Exception as e:
        return Result.custom_error(
            data=e,
            info=f"Error saving database to blob storage: {e}"
        ).set_origin("Blob Dict DB")
initialize(db_path, key, storage_client)

Initializes the database from a location within the blob storage.

Parameters:

Name Type Description Default
db_path str

The base path within the blob storage, e.g., "my_database_blob".

required
key str

The encryption key for the database content.

required
storage_client BlobStorage

An initialized BlobStorage client instance.

required

Returns:

Name Type Description
Result Result

An OK result if successful.

Source code in toolboxv2/mods/DB/blob_instance.py
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
def initialize(self, db_path: str, key: str, storage_client: BlobStorage) -> Result:
    """
    Initializes the database from a location within the blob storage.

    Args:
        db_path (str): The base path within the blob storage,
                       e.g., "my_database_blob".
        key (str): The encryption key for the database content.
        storage_client (BlobStorage): An initialized BlobStorage client instance.

    Returns:
        Result: An OK result if successful.
    """
    self.db_path = db_path.rstrip('/')
    self.key = key
    self.storage_client = storage_client

    print(f"Initializing BlobDB with base path: '{self.db_path}'...")

    # Initialize with empty data - data will be loaded on-demand
    self.data = {}

    print("Successfully initialized database with multi-file storage.")
    return Result.ok().set_origin("Blob Dict DB")

local_instance

load_from_json(filename)

Lädt Daten aus einer JSON-Datei.

:param filename: Der Dateiname oder Pfad der zu ladenden Datei. :return: Die geladenen Daten.

Source code in toolboxv2/mods/DB/local_instance.py
137
138
139
140
141
142
143
144
145
146
147
148
def load_from_json(filename):
    """
    Lädt Daten aus einer JSON-Datei.

    :param filename: Der Dateiname oder Pfad der zu ladenden Datei.
    :return: Die geladenen Daten.
    """
    if not os.path.exists(filename):
        return {'data': ''}

    with open(filename) as file:
        return json.load(file)
save_to_json(data, filename)

Speichert die übergebenen Daten in einer JSON-Datei.

:param data: Die zu speichernden Daten. :param filename: Der Dateiname oder Pfad, in dem die Daten gespeichert werden sollen.

Source code in toolboxv2/mods/DB/local_instance.py
123
124
125
126
127
128
129
130
131
132
133
134
def save_to_json(data, filename):
    """
    Speichert die übergebenen Daten in einer JSON-Datei.

    :param data: Die zu speichernden Daten.
    :param filename: Der Dateiname oder Pfad, in dem die Daten gespeichert werden sollen.
    """
    if not os.path.exists(filename):
        open(filename, 'a').close()

    with open(filename, 'w+') as file:
        json.dump(data, file, indent=4)

reddis_instance

sync_redis_databases(source_url, target_url)

Synchronize keys from the source Redis database to the target Redis database. This function scans all keys in the source DB and uses DUMP/RESTORE to replicate data to the target.

Parameters:

Name Type Description Default
source_url str

The Redis URL of the source database.

required
target_url str

The Redis URL of the target database.

required

Returns:

Name Type Description
int

The number of keys successfully synchronized.

Source code in toolboxv2/mods/DB/reddis_instance.py
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
def sync_redis_databases(source_url, target_url):
    """Synchronize keys from the source Redis database to the target Redis database.
    This function scans all keys in the source DB and uses DUMP/RESTORE to replicate data to the target.

    Args:
        source_url (str): The Redis URL of the source database.
        target_url (str): The Redis URL of the target database.

    Returns:
        int: The number of keys successfully synchronized.
    """
    try:
        src_client = redis.from_url(source_url)
        tgt_client = redis.from_url(target_url)
    except Exception as e:
        print(f"Error connecting to one of the Redis instances: {e}")
        return 0

    total_synced = 0
    cursor = 0
    try:
        while True:
            cursor, keys = src_client.scan(cursor=cursor, count=100)
            for key in keys:
                try:
                    serialized_value = src_client.dump(key)
                    if serialized_value is None:
                        continue
                    # Restore key with TTL=0 and replace existing key
                    tgt_client.restore(key, 0, serialized_value, replace=True)
                    total_synced += 1
                except Exception as e:
                    print(f"Error syncing key {key}: {e}")
            if cursor == 0:
                break
    except Exception as scan_error:
        print(f"Error during scanning keys: {scan_error}")

    print(f"Synced {total_synced} keys from {source_url} to {target_url}")
    return total_synced

tb_adapter

DB

Bases: ABC

Source code in toolboxv2/mods/DB/tb_adapter.py
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
class DB(ABC):
    @abc.abstractmethod
    def get(self, query: str) -> Result:
        """get data"""

    @abc.abstractmethod
    def set(self, query: str, value) -> Result:
        """set data"""

    @abc.abstractmethod
    def append_on_set(self, query: str, value) -> Result:
        """append set data"""

    @abc.abstractmethod
    def delete(self, query: str, matching=False) -> Result:
        """delete data"""

    @abc.abstractmethod
    def if_exist(self, query: str) -> bool:
        """return True if query exists"""

    @abc.abstractmethod
    def exit(self) -> Result:
        """Close DB connection and optional save data"""
append_on_set(query, value) abstractmethod

append set data

Source code in toolboxv2/mods/DB/tb_adapter.py
64
65
66
@abc.abstractmethod
def append_on_set(self, query: str, value) -> Result:
    """append set data"""
delete(query, matching=False) abstractmethod

delete data

Source code in toolboxv2/mods/DB/tb_adapter.py
68
69
70
@abc.abstractmethod
def delete(self, query: str, matching=False) -> Result:
    """delete data"""
exit() abstractmethod

Close DB connection and optional save data

Source code in toolboxv2/mods/DB/tb_adapter.py
76
77
78
@abc.abstractmethod
def exit(self) -> Result:
    """Close DB connection and optional save data"""
get(query) abstractmethod

get data

Source code in toolboxv2/mods/DB/tb_adapter.py
56
57
58
@abc.abstractmethod
def get(self, query: str) -> Result:
    """get data"""
if_exist(query) abstractmethod

return True if query exists

Source code in toolboxv2/mods/DB/tb_adapter.py
72
73
74
@abc.abstractmethod
def if_exist(self, query: str) -> bool:
    """return True if query exists"""
set(query, value) abstractmethod

set data

Source code in toolboxv2/mods/DB/tb_adapter.py
60
61
62
@abc.abstractmethod
def set(self, query: str, value) -> Result:
    """set data"""

ui

api_change_mode(self, request) async

Changes the database mode from a JSON POST body.

Source code in toolboxv2/mods/DB/ui.py
266
267
268
269
270
271
272
273
@export(mod_name=Name, name="api_change_mode", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_change_mode(self, request: RequestData):
    """Changes the database mode from a JSON POST body."""
    data = request.body
    if not data or "mode" not in data:
        return Result.default_user_error("Request body must contain 'mode'.")
    new_mode = data.get("mode", "LC")
    return self.edit_programmable(DatabaseModes.crate(new_mode))
api_delete_key(self, request) async

Deletes a key from a JSON POST body.

Source code in toolboxv2/mods/DB/ui.py
254
255
256
257
258
259
260
261
262
263
@export(mod_name=Name, name="api_delete_key", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_delete_key(self, request: RequestData):
    """Deletes a key from a JSON POST body."""
    data = request.body
    if not data or 'key' not in data:
        return Result.default_user_error("Request body must contain 'key'.")
    key = data['key']
    if not key:
        return Result.default_user_error("Key parameter is required.")
    return self.delete(key)
api_get_all_keys(self, request) async

Returns a list of all keys in the database.

Source code in toolboxv2/mods/DB/ui.py
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
@export(mod_name=Name, name="api_get_all_keys", api=True, request_as_kwarg=True)
async def api_get_all_keys(self, request: RequestData):
    """Returns a list of all keys in the database."""
    if self.data_base:
        keys_result = self.data_base.get('all-k')
        if keys_result.is_error():
            return keys_result

        unwrapped_keys = _unwrap_data(keys_result.get())
        if not isinstance(unwrapped_keys, list):
            self.app.logger.warning(f"get_all_keys did not return a list. Got: {type(unwrapped_keys)}")
            return Result.json(data=[])

        return Result.json(data=sorted(unwrapped_keys))
    return Result.default_internal_error("DB not initialized")
api_get_blob_status(self, request) async

Returns blob storage status - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
@export(mod_name=Name, name="api_get_blob_status", api=True, request_as_kwarg=True)
async def api_get_blob_status(self, request: RequestData):
    """Returns blob storage status - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    try:
        blob_storage = self.app.root_blob_storage
        if not blob_storage:
            return Result.json(data={"status": "unavailable", "servers": []})

        # Get server status
        servers_status = []
        for server in blob_storage.servers:
            try:
                # Basic health check
                status = "online" if server else "offline"
                servers_status.append({
                    "address": str(server),
                    "status": status
                })
            except Exception:
                servers_status.append({
                    "address": str(server),
                    "status": "error"
                })

        return Result.json(data={
            "status": "available",
            "servers": servers_status,
            "storage_dir": getattr(blob_storage, 'storage_directory', 'unknown')
        })
    except Exception as e:
        return Result.default_internal_error(f"Blob status error: {str(e)}")
api_get_cluster_status(self, request) async

Get DB cluster status - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
@export(mod_name=Name, name="api_get_cluster_status", api=True, request_as_kwarg=True)
async def api_get_cluster_status(self, request: RequestData):
    """Get DB cluster status - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    try:
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager
        manager = ClusterManager()
        online_list, server_list = manager.status_all(silent=True)

        instances = []
        for instance_id, instance in manager.instances.items():
            pid, version = instance.read_state()
            instances.append({
                "id": instance_id,
                "port": instance.port,
                "host": instance.host,
                "status": "online" if pid else "offline",
                "pid": pid,
                "version": version
            })

        return Result.json(data={
            "instances": instances,
            "online_count": len(online_list),
            "total_count": len(server_list)
        })
    except Exception as e:
        return Result.default_internal_error(f"Cluster status error: {str(e)}")
api_get_status(self, request) async

Returns the current status of the DB manager.

Source code in toolboxv2/mods/DB/ui.py
195
196
197
198
@export(mod_name=Name, name="api_get_status", api=True, request_as_kwarg=True)
async def api_get_status(self, request: RequestData):
    """Returns the current status of the DB manager."""
    return Result.json(data={"mode": self.mode})
api_get_value(self, request, key) async

Gets a value for a key and returns it as JSON-friendly text.

Source code in toolboxv2/mods/DB/ui.py
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
@export(mod_name=Name, name="api_get_value", api=True, request_as_kwarg=True)
async def api_get_value(self, request: RequestData, key: str):
    """Gets a value for a key and returns it as JSON-friendly text."""
    if not key:
        return Result.default_user_error("Key parameter is required.")
    value_res = self.get(key)
    if value_res.is_error():
        return value_res

    value_unwrapped = _unwrap_data(value_res.get())

    if isinstance(value_unwrapped, bytes):
        try:
            value_str = value_unwrapped.decode('utf-8')
        except UnicodeDecodeError:
            value_str = str(value_unwrapped)
    else:
        value_str = str(value_unwrapped)

    # Simplified for a JSON-focused UI. The client will handle formatting.
    return Result.json(data={"key": key, "value": value_str})
api_list_blob_files(self, request) async

List blob files - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
@export(mod_name=Name, name="api_list_blob_files", api=True, request_as_kwarg=True)
async def api_list_blob_files(self, request: RequestData):
    """List blob files - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    try:
        blob_storage = self.app.root_blob_storage
        if not blob_storage:
            return Result.json(data=[])

        # Get blob IDs
        blob_ids = blob_storage.list_blobs()
        blob_files = []

        for blob_id in blob_ids[:100]:  # Limit to first 100
            try:
                info = blob_storage.get_blob_info(blob_id)
                blob_files.append({
                    "id": blob_id,
                    "size": info.get("size", 0),
                    "created": info.get("created", "unknown"),
                    "encrypted": info.get("encrypted", False)
                })
            except Exception:
                blob_files.append({
                    "id": blob_id,
                    "size": 0,
                    "created": "unknown",
                    "encrypted": False
                })

        return Result.json(data=blob_files)
    except Exception as e:
        return Result.default_internal_error(f"Blob listing error: {str(e)}")
api_manage_cluster(self, request) async

Manage cluster instances - admin/trusted only.

Source code in toolboxv2/mods/DB/ui.py
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
@export(mod_name=Name, name="api_manage_cluster", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_manage_cluster(self, request: RequestData):
    """Manage cluster instances - admin/trusted only."""
    if not await _is_admin_or_trusted(self.app, request):
        return Result.default_user_error("Access denied")

    data = request.body
    if not data or 'action' not in data:
        return Result.default_user_error("Request body must contain 'action'.")

    action = data['action']
    instance_id = data.get('instance_id')

    try:
        from toolboxv2.utils.clis.db_cli_manager import ClusterManager, get_executable_path
        manager = ClusterManager()

        if action == 'start':
            executable_path = get_executable_path()
            if instance_id:
                result = manager.start(executable_path, "current", instance_id)
            else:
                result = manager.start_all(executable_path, "current")
            return Result.ok(data=f"Start command executed: {result}")

        elif action == 'stop':
            if instance_id:
                result = manager.stop(instance_id)
            else:
                result = manager.stop_all()
            return Result.ok(data=f"Stop command executed: {result}")

        elif action == 'restart':
            if instance_id:
                manager.stop(instance_id)
                executable_path = get_executable_path()
                result = manager.start(executable_path, "current", instance_id)
            else:
                manager.stop_all()
                executable_path = get_executable_path()
                result = manager.start_all(executable_path, "current")
            return Result.ok(data=f"Restart command executed: {result}")

        else:
            return Result.default_user_error("Invalid action")

    except Exception as e:
        return Result.default_internal_error(f"Cluster management error: {str(e)}")
api_set_value(self, request) async

Sets a key-value pair from a JSON POST body.

Source code in toolboxv2/mods/DB/ui.py
241
242
243
244
245
246
247
248
249
250
251
@export(mod_name=Name, name="api_set_value", api=True, api_methods=['POST'], request_as_kwarg=True)
async def api_set_value(self, request: RequestData):
    """Sets a key-value pair from a JSON POST body."""
    data = request.body
    if not data or 'key' not in data or 'value' not in data:
        return Result.default_user_error("Request body must contain 'key' and 'value'.")
    key = data['key']
    value = data['value']
    if not key:
        return Result.default_user_error("Key cannot be empty.")
    return self.set(key, value)
db_manager_ui(**kwargs)

Serves the refactored, JSON-focused UI for the DB Manager.

Source code in toolboxv2/mods/DB/ui.py
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
@export(mod_name=Name, name="ui", api=True, state=False)
def db_manager_ui(**kwargs):
    """Serves the refactored, JSON-focused UI for the DB Manager."""
    html_content = """
    <!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>DB Manager</title>
        <style>
            :root {
                --font-family-sans: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif;
                --font-family-mono: "SF Mono", "Menlo", "Monaco", "Courier New", Courier, monospace;
                --color-bg: #f8f9fa;
                --color-panel-bg: #ffffff;
                --color-border: #dee2e6;
                --color-text: #212529;
                --color-text-muted: #6c757d;
                --color-primary: #0d6efd;
                --color-primary-hover: #0b5ed7;
                --color-danger: #dc3545;
                --color-danger-hover: #bb2d3b;
                --color-key-folder-icon: #f7b731;
                --color-key-file-icon: #adb5bd;
                --color-key-hover-bg: #e9ecef;
                --color-key-selected-bg: #0d6efd;
                --color-key-selected-text: #ffffff;
                --shadow-sm: 0 1px 2px 0 rgba(0, 0, 0, 0.05);
                --radius: 0.375rem;
            }

            /* Basic styles */
            * { box-sizing: border-box; }
            html { font-size: 16px; }

            body {
                font-family: var(--font-family-sans);
                background-color: var(--color-bg);
                color: var(--color-text);
                margin: 0;
                padding: 1rem;
                display: flex;
                flex-direction: column;
                height: 100vh;
            }

            /* Main layout */
            .db-manager-container { display: flex; flex-direction: column; height: 100%; gap: 1rem; }
            .db-header { display: flex; justify-content: space-between; align-items: center; padding-bottom: 1rem; border-bottom: 1px solid var(--color-border); flex-shrink: 0; }
            .db-main-content { display: flex; gap: 1rem; flex: 1; min-height: 0; }

            /* Panels */
            .db-panel { background-color: var(--color-panel-bg); border: 1px solid var(--color-border); border-radius: var(--radius); box-shadow: var(--shadow-sm); display: flex; flex-direction: column; min-height: 0; }
            .key-panel { width: 350px; min-width: 250px; max-width: 450px; }
            .editor-panel, .placeholder-panel { flex-grow: 1; }
            .panel-header { display: flex; justify-content: space-between; align-items: center; padding: 0.75rem 1rem; border-bottom: 1px solid var(--color-border); flex-shrink: 0; }
            .panel-header h2 { font-size: 1.1rem; margin: 0; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }

            /* Controls */
            select, input[type="text"], textarea, button { font-size: 1rem; }
            select, input[type="text"] { background-color: var(--color-bg); color: var(--color-text); border: 1px solid var(--color-border); border-radius: var(--radius); padding: 0.5rem 0.75rem; }
            select:focus, input[type="text"]:focus, textarea:focus { outline: 2px solid var(--color-primary); outline-offset: -1px; }
            button { border: none; border-radius: var(--radius); padding: 0.5rem 1rem; font-weight: 500; cursor: pointer; transition: background-color 0.2s; }
            button.primary { background-color: var(--color-primary); color: white; }
            button.primary:hover { background-color: var(--color-primary-hover); }
            button.danger { background-color: var(--color-danger); color: white; }
            button.danger:hover { background-color: var(--color-danger-hover); }
            .header-actions { display: flex; gap: 0.5rem; }

            /* Key Tree View */
            #keySearchInput { width: calc(100% - 2rem); margin: 1rem; flex-shrink: 0; }
            .key-tree-container { font-family: var(--font-family-mono); font-size: 0.9rem; padding: 0 0.5rem 1rem; overflow-y: auto; flex: 1; min-height: 0; }
            .key-tree-container ul { list-style: none; padding-left: 0; margin: 0; }
            .key-tree-container li { padding-left: 20px; position: relative; }
            .node-label { display: flex; align-items: center; padding: 4px 8px; cursor: pointer; border-radius: 4px; word-break: break-all; user-select: none; }
            .node-label:hover { background-color: var(--color-key-hover-bg); }
            .node-label.selected { background-color: var(--color-key-selected-bg); color: var(--color-key-selected-text); }
            .node-label.selected .node-icon { color: var(--color-key-selected-text) !important; }
            .node-icon { width: 20px; text-align: center; margin-right: 5px; flex-shrink: 0; }
            .tree-folder > .node-label .node-icon { color: var(--color-key-folder-icon); font-style: normal; }
            .tree-folder > .node-label .node-icon::before { content: '▸'; display: inline-block; transition: transform 0.15s ease-in-out; }
            .tree-folder.open > .node-label .node-icon::before { transform: rotate(90deg); }
            .tree-leaf > .node-label .node-icon { color: var(--color-key-file-icon); }
            .tree-leaf > .node-label .node-icon::before { content: '•'; }
            .tree-children { display: none; }
            .tree-folder.open > .tree-children { display: block; }

            /* Editor Panel */
            .editor-toolbar { display: flex; gap: 1rem; align-items: center; padding: 0.75rem 1rem; border-bottom: 1px solid var(--color-border); flex-shrink: 0; }
            #valueEditor { flex: 1; width: 100%; min-height: 0; border: none; resize: none; font-family: var(--font-family-mono); font-size: 0.95rem; line-height: 1.5; padding: 1rem; background: transparent; color: var(--color-text); }
            #valueEditor:focus { outline: none; }

            /* Placeholder and Utility */
            .placeholder-panel { display: flex; flex-direction: column; align-items: center; justify-content: center; color: var(--color-text-muted); text-align: center; }
            .hidden { display: none !important; }
            .key-tree-container p.status-message { padding: 1rem; margin: 0; color: var(--color-text-muted); text-align: center; }

            /* Custom Scrollbars */
            .key-tree-container::-webkit-scrollbar, #valueEditor::-webkit-scrollbar { width: 8px; height: 8px; }
            .key-tree-container::-webkit-scrollbar-track, #valueEditor::-webkit-scrollbar-track { background: transparent; }
            .key-tree-container::-webkit-scrollbar-thumb, #valueEditor::-webkit-scrollbar-thumb { background-color: var(--color-border); border-radius: 4px; }
            .key-tree-container::-webkit-scrollbar-thumb:hover, #valueEditor::-webkit-scrollbar-thumb:hover { background-color: var(--color-text-muted); }
            #valueEditor::-webkit-scrollbar-corner { background: transparent; }

            /* Responsive */
            @media (max-width: 768px) {
                body { padding: 0.5rem; }
                .db-main-content { flex-direction: column; }
                .key-panel { width: 100%; max-height: 40vh; }
            }

            /* New styles for enhanced features */
            .tab-container {
                display: flex;
                border-bottom: 1px solid var(--color-border);
                margin-bottom: 1rem;
            }

            .tab-button {
                padding: 0.75rem 1.5rem;
                border: none;
                background: none;
                cursor: pointer;
                border-bottom: 2px solid transparent;
                font-weight: 500;
            }

            .tab-button.active {
                border-bottom-color: var(--color-primary);
                color: var(--color-primary);
            }

            .tab-content {
                display: none;
            }

            .tab-content.active {
                display: block;
            }

            .status-grid {
                display: grid;
                grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
                gap: 1rem;
                margin-bottom: 1rem;
            }

            .status-card {
                background: var(--color-panel-bg);
                border: 1px solid var(--color-border);
                border-radius: var(--radius);
                padding: 1rem;
            }

            .status-indicator {
                display: inline-block;
                width: 8px;
                height: 8px;
                border-radius: 50%;
                margin-right: 0.5rem;
            }

            .status-online { background-color: #28a745; }
            .status-offline { background-color: #dc3545; }
            .status-error { background-color: #ffc107; }

            .blob-file-list {
                max-height: 400px;
                overflow-y: auto;
                border: 1px solid var(--color-border);
                border-radius: var(--radius);
            }

            .blob-file-item {
                display: flex;
                justify-content: space-between;
                align-items: center;
                padding: 0.75rem;
                border-bottom: 1px solid var(--color-border);
            }

            .blob-file-item:last-child {
                border-bottom: none;
            }

            .cluster-controls {
                display: flex;
                gap: 0.5rem;
                margin-bottom: 1rem;
            }

            .instance-list {
                display: grid;
                gap: 0.5rem;
            }

            .instance-item {
                display: flex;
                justify-content: space-between;
                align-items: center;
                padding: 0.75rem;
                background: var(--color-panel-bg);
                border: 1px solid var(--color-border);
                border-radius: var(--radius);
            }
        </style>
    </head>
    <body>
        <div id="dbManagerContainer" class="db-manager-container">
            <header class="db-header">
                <h1>DB Manager</h1>
                <div class="db-mode-selector">
                    <label for="modeSelect">Mode:</label>
                    <select id="modeSelect">
                        <option value="LC">Local Dict</option>
                        <option value="CB">Cloud Blob</option>
                        <option value="LR">Local Redis</option>
                        <option value="RR">Remote Redis</option>
                    </select>
                </div>
            </header>

            <div class="tab-container">
                <button class="tab-button active" data-tab="database">Database</button>
                <button class="tab-button" data-tab="blob-storage" id="blobTab" style="display:none;">Blob Storage</button>
                <button class="tab-button" data-tab="cluster" id="clusterTab" style="display:none;">Cluster</button>
            </div>

            <main class="db-main-content">
                <!-- Database Tab -->
                <div id="database-tab" class="tab-content active">
                    <aside id="keyPanel" class="db-panel key-panel">
                        <div class="panel-header">
                            <h2>Keys</h2>
                            <div class="header-actions">
                                <button id="addKeyBtn" title="Add New Key">+</button>
                                <button id="refreshKeysBtn" title="Refresh Keys">🔄</button>
                            </div>
                        </div>
                        <input type="text" id="keySearchInput" placeholder="Search keys...">
                        <div id="keyTreeContainer" class="key-tree-container"></div>
                    </aside>
                    <section id="editorPanel" class="db-panel editor-panel hidden">
                        <div class="panel-header">
                            <h2 id="selectedKey"></h2>
                            <div class="header-actions">
                                <button id="saveBtn" class="primary">Save</button>
                                <button id="deleteBtn" class="danger">Delete</button>
                            </div>
                        </div>
                        <div class="editor-toolbar">
                            <button id="formatBtn">Format JSON</button>
                        </div>
                        <textarea id="valueEditor" placeholder="Select a key to view its value..."></textarea>
                    </section>
                    <section id="placeholderPanel" class="db-panel editor-panel placeholder-panel">
                        <h3>Select a key to get started</h3>
                        <p>Or click the '+' button to add a new one.</p>
                    </section>
                </div>

                <!-- Blob Storage Tab -->
                <div id="blob-storage-tab" class="tab-content">
                    <div class="status-grid">
                        <div class="status-card">
                            <h3>Blob Storage Status</h3>
                            <div id="blobStorageStatus">Loading...</div>
                        </div>
                        <div class="status-card">
                            <h3>Server Health</h3>
                            <div id="serverHealth">Loading...</div>
                        </div>
                    </div>
                    <div class="db-panel">
                        <div class="panel-header">
                            <h2>Blob Files</h2>
                            <div class="header-actions">
                                <button id="refreshBlobsBtn">🔄 Refresh</button>
                            </div>
                        </div>
                        <div id="blobFileList" class="blob-file-list">Loading...</div>
                    </div>
                </div>

                <!-- Cluster Tab -->
                <div id="cluster-tab" class="tab-content">
                    <div class="cluster-controls">
                        <button id="startAllBtn" class="primary">Start All</button>
                        <button id="stopAllBtn" class="danger">Stop All</button>
                        <button id="restartAllBtn">Restart All</button>
                        <button id="refreshClusterBtn">🔄 Refresh</button>
                    </div>
                    <div class="db-panel">
                        <div class="panel-header">
                            <h2>Cluster Instances</h2>
                        </div>
                        <div id="instanceList" class="instance-list">Loading...</div>
                    </div>
                </div>
            </main>
        </div>
        <script>
        (() => {
            "use strict";
            const API_NAME = "DB";
            let isAdminUser = false;

            class DBManager {
                constructor() {
                    this.cache = {
                        keys: [],
                        selectedKey: null,
                        blobFiles: [],
                        clusterStatus: null
                    };
                    this.dom = {
                        modeSelect: document.getElementById('modeSelect'),
                        keySearchInput: document.getElementById('keySearchInput'),
                        keyTreeContainer: document.getElementById('keyTreeContainer'),
                        editorPanel: document.getElementById('editorPanel'),
                        placeholderPanel: document.getElementById('placeholderPanel'),
                        selectedKey: document.getElementById('selectedKey'),
                        valueEditor: document.getElementById('valueEditor'),
                        addKeyBtn: document.getElementById('addKeyBtn'),
                        refreshKeysBtn: document.getElementById('refreshKeysBtn'),
                        saveBtn: document.getElementById('saveBtn'),
                        deleteBtn: document.getElementById('deleteBtn'),
                        formatBtn: document.getElementById('formatBtn'),
                        tabButtons: document.querySelectorAll('.tab-button'),
                        tabContents: document.querySelectorAll('.tab-content'),
                        blobTab: document.getElementById('blobTab'),
                        clusterTab: document.getElementById('clusterTab'),
                        refreshBlobsBtn: document.getElementById('refreshBlobsBtn'),
                        blobFileList: document.getElementById('blobFileList'),
                        blobStorageStatus: document.getElementById('blobStorageStatus'),
                        serverHealth: document.getElementById('serverHealth'),
                        instanceList: document.getElementById('instanceList'),
                        startAllBtn: document.getElementById('startAllBtn'),
                        stopAllBtn: document.getElementById('stopAllBtn'),
                        restartAllBtn: document.getElementById('restartAllBtn'),
                        refreshClusterBtn: document.getElementById('refreshClusterBtn')
                    };
                    this.init();
                }

                async init() {
                    this.addEventListeners();
                    await this.checkAdminAccess();
                    await this.loadInitialStatus();
                    await this.loadKeys();
                }

                async checkAdminAccess() {
                    try {
                        const res = await this.apiRequest('api_get_blob_status', null, 'GET');
                        if (!res.error) {
                            isAdminUser = true;
                            this.dom.blobTab.style.display = 'block';
                            this.dom.clusterTab.style.display = 'block';
                        }
                    } catch (e) {
                        // User doesn't have admin access
                        isAdminUser = false;
                    }
                }

                addEventListeners() {
                    // Tab switching
                    this.dom.tabButtons.forEach(button => {
                        button.addEventListener('click', (e) => {
                            const tabName = e.target.dataset.tab;
                            this.switchTab(tabName);
                        });
                    });

                    // Blob storage events
                    if (this.dom.refreshBlobsBtn) {
                        this.dom.refreshBlobsBtn.addEventListener('click', () => this.loadBlobFiles());
                    }

                    // Cluster events
                    if (this.dom.startAllBtn) {
                        this.dom.startAllBtn.addEventListener('click', () => this.manageCluster('start'));
                        this.dom.stopAllBtn.addEventListener('click', () => this.manageCluster('stop'));
                        this.dom.restartAllBtn.addEventListener('click', () => this.manageCluster('restart'));
                        this.dom.refreshClusterBtn.addEventListener('click', () => this.loadClusterStatus());
                    }

                    this.dom.refreshKeysBtn.addEventListener('click', () => this.loadKeys());
                    this.dom.addKeyBtn.addEventListener('click', () => this.showAddKeyModal());
                    this.dom.saveBtn.addEventListener('click', () => this.saveValue());
                    this.dom.deleteBtn.addEventListener('click', () => this.confirmDeleteKey());
                    this.dom.formatBtn.addEventListener('click', () => this.formatJson());
                    this.dom.keySearchInput.addEventListener('input', (e) => this.renderKeyTree(e.target.value));
                    this.dom.modeSelect.addEventListener('change', (e) => this.changeMode(e.target.value));

                    this.dom.keyTreeContainer.addEventListener('click', (e) => {
                        const label = e.target.closest('.node-label');
                        if (!label) return;
                        const node = label.parentElement;
                        if (node.classList.contains('tree-folder')) {
                            node.classList.toggle('open');
                        } else if (node.dataset.key) {
                            this.selectKey(node.dataset.key);
                        }
                    });
                }

                async apiRequest(endpoint, payload = null, method = 'POST') {
                    if (!window.TB?.api?.request) {
                        console.error("TB.api not available!");
                        return { error: true, message: "TB.api not available" };
                    }
                    try {
                        const url = (method === 'GET' && payload) ? `${endpoint}?${new URLSearchParams(payload)}` : endpoint;
                        const body = (method !== 'GET') ? payload : null;
                        const response = await window.TB.api.request(API_NAME, url, body, method);

                        if (response.error && response.error !== 'none') {
                            const errorMsg = response.info?.help_text || response.error;
                            console.error(`API Error on ${endpoint}:`, errorMsg, response);
                            if (window.TB?.ui?.Toast) TB.ui.Toast.showError(errorMsg, { duration: 5000 });
                            return { error: true, message: errorMsg, data: response.get() };
                        }
                        return { error: false, data: response.get() };
                    } catch (err) {
                        console.error("Framework/Network Error:", err);
                        if (window.TB?.ui?.Toast) TB.ui.Toast.showError("Application or network error.", { duration: 5000 });
                        return { error: true, message: "Network error" };
                    }
                }

                async loadInitialStatus() {
                    const res = await this.apiRequest('api_get_status', null, 'GET');
                    if (!res.error) this.dom.modeSelect.value = res.data.mode;
                }

                async loadKeys() {
                    this.setStatusMessage('Loading keys...');
                    const res = await this.apiRequest('api_get_all_keys', null, 'GET');
                    if (!res.error) {
                        this.cache.keys = res.data || [];
                        this.renderKeyTree();
                    } else {
                        this.setStatusMessage('Failed to load keys.', true);
                    }
                }

                renderKeyTree(filter = '') {
                    const treeData = {};
                    const filteredKeys = this.cache.keys.filter(k => k.toLowerCase().includes(filter.toLowerCase().trim()));

                    for (const key of filteredKeys) {
                        let currentLevel = treeData;
                        const parts = key.split(':');
                        for (let i = 0; i < parts.length; i++) {
                            const part = parts[i];
                            if (!part) continue; // Skip empty parts from keys like "a::b"
                            const isLeaf = i === parts.length - 1;

                            if (!currentLevel[part]) {
                                currentLevel[part] = { _children: {} };
                            }
                            if (isLeaf) {
                                currentLevel[part]._fullKey = key;
                            }
                            currentLevel = currentLevel[part]._children;
                        }
                    }

                    const treeHtml = this.buildTreeHtml(treeData);
                    if (treeHtml) {
                        this.dom.keyTreeContainer.innerHTML = `<ul class="key-tree">${treeHtml}</ul>`;
                        // Re-select the key if it's still visible
                        if (this.cache.selectedKey) {
                             const nodeEl = this.dom.keyTreeContainer.querySelector(`[data-key="${this.cache.selectedKey}"] .node-label`);
                             if(nodeEl) nodeEl.classList.add('selected');
                        }
                    } else {
                         this.setStatusMessage(filter ? 'No keys match your search.' : 'No keys found.');
                    }
                }

                buildTreeHtml(node) {
                    return Object.keys(node).sort().map(key => {
                        const childNode = node[key];
                        const isFolder = Object.keys(childNode._children).length > 0;

                        if (isFolder) {
                            return `<li class="tree-folder" ${childNode._fullKey ? `data-key="${childNode._fullKey}"`: ''}>
                                        <div class="node-label"><i class="node-icon"></i>${key}</div>
                                        <ul class="tree-children">${this.buildTreeHtml(childNode._children)}</ul>
                                    </li>`;
                        } else {
                            return `<li class="tree-leaf" data-key="${childNode._fullKey}">
                                        <div class="node-label"><i class="node-icon"></i>${key}</div>
                                    </li>`;
                        }
                    }).join('');
                }

                async selectKey(key) {
                    if (!key) return;
                    this.showEditor(true);
                    this.cache.selectedKey = key;

                    document.querySelectorAll('.node-label.selected').forEach(el => el.classList.remove('selected'));
                    const nodeEl = this.dom.keyTreeContainer.querySelector(`[data-key="${key}"] > .node-label`);
                    if (nodeEl) nodeEl.classList.add('selected');

                    this.dom.selectedKey.textContent = key;
                    this.dom.selectedKey.title = key;
                    this.dom.valueEditor.value = "Loading...";

                    const res = await this.apiRequest('api_get_value', { key }, 'GET');
                    this.dom.valueEditor.value = res.error ? `Error: ${res.message}` : res.data.value;
                    if (!res.error) this.formatJson(false); // Auto-format if it's valid JSON, without showing an error
                }

                async saveValue() {
                    if (!this.cache.selectedKey) return;
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show("Saving...");
                    const res = await this.apiRequest('api_set_value', {
                        key: this.cache.selectedKey,
                        value: this.dom.valueEditor.value
                    });
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();
                    if (!res.error && window.TB?.ui?.Toast) TB.ui.Toast.showSuccess("Key saved successfully!");
                }

                async confirmDeleteKey() {
                    if (!this.cache.selectedKey) return;
                    if (!window.TB?.ui?.Modal) {
                        if(confirm(`Delete key "${this.cache.selectedKey}"?`)) this.deleteKey();
                        return;
                    }
                    TB.ui.Modal.confirm({
                        title: 'Delete Key?',
                        content: `Are you sure you want to delete the key "<strong>${this.cache.selectedKey}</strong>"?<br/>This action cannot be undone.`,
                        confirmButtonText: 'Delete',
                        confirmButtonVariant: 'danger',
                        onConfirm: () => this.deleteKey()
                    });
                }

                async deleteKey() {
                    const keyToDelete = this.cache.selectedKey;
                    if (!keyToDelete) return;
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show("Deleting...");
                    const res = await this.apiRequest('api_delete_key', { key: keyToDelete });
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();

                    if (!res.error) {
                        if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess(`Key "${keyToDelete}" deleted.`);
                        this.cache.selectedKey = null;
                        this.showEditor(false);
                        this.loadKeys(); // Refresh the key list
                    }
                }

                formatJson(showErrorToast = true) {
                    try {
                        const currentVal = this.dom.valueEditor.value.trim();
                        if (!currentVal) return;
                        const formatted = JSON.stringify(JSON.parse(currentVal), null, 2);
                        this.dom.valueEditor.value = formatted;
                    } catch (e) {
                        if (showErrorToast && window.TB?.ui?.Toast) {
                            TB.ui.Toast.showWarning("Value is not valid JSON.", { duration: 3000 });
                        }
                    }
                }

                showAddKeyModal() {
                     if (!window.TB?.ui?.Modal) { alert("Add Key modal not available."); return; }
                     TB.ui.Modal.show({
                        title: 'Add New Key',
                        content: `<input type="text" id="newKeyInput" placeholder="Enter new key name (e.g., app:settings:user)" style="width: 100%; margin-bottom: 1rem;"/>
                                  <textarea id="newValueInput" placeholder='Enter value (e.g., {"theme": "dark"})' style="width: 100%; height: 150px; font-family: var(--font-family-mono);"></textarea>`,
                        onOpen: (modal) => document.getElementById('newKeyInput').focus(),
                        buttons: [{
                            text: 'Save', variant: 'primary',
                            action: async (modal) => {
                                const newKey = document.getElementById('newKeyInput').value.trim();
                                const newValue = document.getElementById('newValueInput').value;
                                if (!newKey) { if (window.TB?.ui?.Toast) TB.ui.Toast.showError("Key name cannot be empty."); return; }
                                modal.close();
                                if (window.TB?.ui.Loader) TB.ui.Loader.show("Saving...");
                                const res = await this.apiRequest('api_set_value', { key: newKey, value: newValue });
                                if (window.TB?.ui.Loader) TB.ui.Loader.hide();
                                if (!res.error) {
                                    if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess("New key created!");
                                    await this.loadKeys();
                                    this.selectKey(newKey);
                                }
                            }
                        }, { text: 'Cancel', action: (modal) => modal.close() }]
                    });
                }

                async changeMode(newMode) {
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show(`Switching to ${newMode}...`);
                    const res = await this.apiRequest('api_change_mode', { mode: newMode });
                    if (!res.error) {
                       this.cache.selectedKey = null;
                       this.showEditor(false);
                       await this.loadKeys();
                       if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess(`Switched to ${newMode} mode.`);
                    } else {
                       if (window.TB?.ui?.Toast) TB.ui.Toast.showError(`Failed to switch mode.`);
                       await this.loadInitialStatus(); // Revert dropdown to actual status
                    }
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();
                }

                showEditor(show) {
                    this.dom.editorPanel.classList.toggle('hidden', !show);
                    this.dom.placeholderPanel.classList.toggle('hidden', show);
                }

                setStatusMessage(message, isError = false) {
                    this.dom.keyTreeContainer.innerHTML = `<p class="status-message" style="${isError ? 'color: var(--color-danger);' : ''}">${message}</p>`;
                }

                switchTab(tabName) {
                    // Update tab buttons
                    this.dom.tabButtons.forEach(btn => btn.classList.remove('active'));
                    document.querySelector(`[data-tab="${tabName}"]`).classList.add('active');

                    // Update tab content
                    this.dom.tabContents.forEach(content => content.classList.remove('active'));
                    document.getElementById(`${tabName}-tab`).classList.add('active');

                    // Load tab-specific data
                    if (tabName === 'blob-storage' && isAdminUser) {
                        this.loadBlobStatus();
                        this.loadBlobFiles();
                    } else if (tabName === 'cluster' && isAdminUser) {
                        this.loadClusterStatus();
                    }
                }

                async loadBlobStatus() {
                    const res = await this.apiRequest('api_get_blob_status', null, 'GET');
                    if (!res.error) {
                        const data = res.data;
                        this.dom.blobStorageStatus.innerHTML = `
                            <div><span class="status-indicator status-${data.status === 'available' ? 'online' : 'offline'}"></span>${data.status}</div>
                            <div>Storage: ${data.storage_dir}</div>
                        `;

                        const serverHtml = data.servers.map(server =>
                            `<div><span class="status-indicator status-${server.status}"></span>${server.address}</div>`
                        ).join('');
                        this.dom.serverHealth.innerHTML = serverHtml || 'No servers';
                    }
                }

                async loadBlobFiles() {
                    const res = await this.apiRequest('api_list_blob_files', null, 'GET');
                    if (!res.error) {
                        this.cache.blobFiles = res.data;
                        this.renderBlobFiles();
                    }
                }

                renderBlobFiles() {
                    if (this.cache.blobFiles.length === 0) {
                        this.dom.blobFileList.innerHTML = '<div class="blob-file-item">No blob files found</div>';
                        return;
                    }

                    const html = this.cache.blobFiles.map(file => `
                        <div class="blob-file-item">
                            <div>
                                <strong>${file.id}</strong>
                                <div style="font-size: 0.875rem; color: var(--color-text-muted);">
                                    ${this.formatBytes(file.size)} • ${file.created} • ${file.encrypted ? '🔒 Encrypted' : '🔓 Plain'}
                                </div>
                            </div>
                        </div>
                    `).join('');

                    this.dom.blobFileList.innerHTML = html;
                }

                async loadClusterStatus() {
                    const res = await this.apiRequest('api_get_cluster_status', null, 'GET');
                    if (!res.error) {
                        this.cache.clusterStatus = res.data;
                        this.renderClusterStatus();
                    }
                }

                renderClusterStatus() {
                    if (!this.cache.clusterStatus) return;

                    const html = this.cache.clusterStatus.instances.map(instance => `
                        <div class="instance-item">
                            <div>
                                <strong>${instance.id}</strong>
                                <div style="font-size: 0.875rem; color: var(--color-text-muted);">
                                    ${instance.host}:${instance.port} • PID: ${instance.pid || 'N/A'} • ${instance.version || 'Unknown'}
                                </div>
                            </div>
                            <div>
                                <span class="status-indicator status-${instance.status}"></span>
                                ${instance.status}
                            </div>
                        </div>
                    `).join('');

                    this.dom.instanceList.innerHTML = html;
                }

                async manageCluster(action) {
                    if (window.TB?.ui?.Loader) TB.ui.Loader.show(`${action}ing cluster...`);
                    const res = await this.apiRequest('api_manage_cluster', { action });
                    if (window.TB?.ui?.Loader) TB.ui.Loader.hide();

                    if (!res.error) {
                        if (window.TB?.ui?.Toast) TB.ui.Toast.showSuccess(`Cluster ${action} completed`);
                        setTimeout(() => this.loadClusterStatus(), 2000);
                    }
                }

                formatBytes(bytes) {
                    if (bytes === 0) return '0 Bytes';
                    const k = 1024;
                    const sizes = ['Bytes', 'KB', 'MB', 'GB'];
                    const i = Math.floor(Math.log(bytes) / Math.log(k));
                    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
                }
            }

            function onTbReady() { new DBManager(); }
            if (window.TB?.events) {
                if (window.TB.config?.get('appRootId')) {
                    onTbReady();
                } else {
                    window.TB.events.on('tbjs:initialized', onTbReady, { once: true });
                }
            } else {
                document.addEventListener('tbjs:initialized', onTbReady, { once: true });
            }
        </script>
    </body>
    </html>
    """
    app = get_app(Name)
    try:
        # Prepend the web context to include necessary framework scripts (like TB.js)
        web_context = app.web_context()
        return Result.html(web_context + html_content)
    except Exception:
        # Fallback in case web_context is not available
        return Result.html(html_content)

EventManager

module

EventManagerClass
Source code in toolboxv2/mods/EventManager/module.py
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
class EventManagerClass:
    events: set[Event] = set()
    source_id: str
    _name: str
    _identification: str

    routes_client: dict[str, ProxyRout] = {}
    routers_servers: dict[str, DaemonRout] = {}
    routers_servers_tasks: list[Any] = []
    routers_servers_tasks_running_flag: bool = False

    receiver_que: queue.Queue
    response_que: queue.Queue

    def add_c_route(self, name, route: ProxyRout):
        self.routes_client[name] = route

    async def receive_all_client_data(self):

        close_connections = []
        add_ev = []
        for name, client in self.routes_client.items():
            if client.client is None or not client.client.get('alive', False):
                close_connections.append(name)
                continue
            data = client.r

            if isinstance(data, str) and data == "No data":
                continue
            elif isinstance(data, EventID) and len(data.get_source()) != 0:
                await self.trigger_event(data)
            elif isinstance(data, EventID) and len(data.get_source()) == 0:
                print(f"Event returned {data.payload}")
                self.response_que.put(data)
            elif isinstance(data,
                            dict) and 'error' in data and 'origin' in data and 'result' in data and 'info' in data:

                self.response_que.put(Result.result_from_dict(**data).print())
            elif isinstance(data,
                            dict) and 'source' in data and 'path' in data and 'ID' in data and 'identifier' in data:
                del data['identifier']
                ev_id = EventID(**data)
                await self.trigger_event(ev_id)
            elif isinstance(data, Event):
                print("Event:", str(data.event_id), data.name)
                add_ev.append(data)
            elif isinstance(data, Result):
                self.response_que.put(data.print())
            else:
                print(f"Unknown Data {data}")

        for ev in add_ev:
            await self.register_event(ev)

        for client_name in close_connections:
            print(f"Client {client_name} closing connection")
            self.remove_c_route(client_name)

    def remove_c_route(self, name):
        self.routes_client[name].close()
        del self.routes_client[name]

    def crate_rout(self, source, addr=None):
        if addr is None:
            addr = ('0.0.0.0', 6588)
        host, port = addr
        if isinstance(port, str):
            port = int(port)
        return Rout(
            _from=self.source_id,
            _to=source,
            _from_port=int(os.getenv("TOOLBOXV2_BASE_PORT", 6588)),
            _from_host=os.getenv("TOOLBOXV2_BASE_HOST"),
            _to_port=port,
            _to_host=host,
            routing_function=self.routing_function_router,
        )

    def __init__(self, source_id, _identification="PN"):
        self.bo = False
        self.running = False
        self.source_id = source_id
        self.receiver_que = queue.Queue()
        self.response_que = queue.Queue()
        self._identification = _identification
        self._name = self._identification + '-' + str(uuid.uuid4()).split('-')[1]
        self.routes = {}
        self.logger = get_logger()

    @property
    def identification(self) -> str:
        return self._identification

    @identification.setter
    def identification(self, _identification: str):
        self.stop()
        self._identification = _identification
        self._name = self._identification + '-' + str(uuid.uuid4()).split('-')[1]

    async def identity_post_setter(self):

        do_reconnect = len(list(self.routers_servers.keys())) > 0
        if self._identification == "P0":
            await self.add_server_route(self._identification, ('0.0.0.0', 6568))
        if self._identification == "P0|S0":
            await self.add_server_route(self._identification, ('0.0.0.0', 6567))

        await asyncio.sleep(0.1)
        self.start()
        await asyncio.sleep(0.1)
        if do_reconnect:
            self.reconnect("ALL")

    async def open_connection_server(self, port):
        await self.add_server_route(self._identification, ('0.0.0.0', port))

    def start(self):
        self.running = True
        threading.Thread(target=async_test(self.receiver), daemon=True).start()

    def make_event_from_fuction(self, fuction, name, *args, source_types=SourceTypes.F,
                                scope=Scope.local,
                                exec_in=ExecIn.local,
                                threaded=False, **kwargs):

        return Event(source=fuction,
                     name=name,
                     event_id=EventID.crate_with_source(self.source_id), args=args,
                     kwargs_=kwargs,
                     source_types=source_types,
                     scope=scope,
                     exec_in=exec_in,
                     threaded=threaded,
                     )

    async def add_client_route(self, source_id, addr):
        if source_id in self.routes_client:
            if self.routes_client[source_id].client is None or not self.routes_client[source_id].client.get('alive'):
                await self.routes_client[source_id].reconnect()
                return True
            print("Already connected")
            return False
        try:
            pr = await ProxyRout.toProxy(rout=self.crate_rout(source_id, addr=addr), name=source_id)
            await asyncio.sleep(0.1)
            await pr.client.get('sender')({"id": self._identification,
                                           "continue": False,
                                           "key": os.getenv('TB_R_KEY', 'root@remote')})
            await asyncio.sleep(0.1)
            self.add_c_route(source_id, pr)
            return True
        except Exception as e:
            print(f"Check the port {addr} Sever likely not Online : {e}")
            return False

    async def add_mini_client(self, name: str, addr: tuple[str, int]):

        mini_proxy = await ProxyRout(class_instance=None, timeout=15, app=get_app(),
                                     remote_functions=[""], peer=False, name=name, do_connect=False)

        async def _(x):
            return await self.routers_servers[self._identification].send(x, addr)

        mini_proxy.put_data = _
        mini_proxy.connect = lambda *x, **_: None
        mini_proxy.reconnect = lambda *x, **_: None
        mini_proxy.close = lambda *x, **_: None
        mini_proxy.client = {'alive': True}
        mini_proxy.r = "No data"
        self.routes_client[name] = mini_proxy

    async def on_register(self, id_, data):
        try:
            if "unknown" not in self.routes:
                self.routes["unknown"] = {}

            if id_ != "new_con" and 'id' in data:
                id_data = data.get('id')
                id_ = eval(id_)
                c_host, c_pot = id_
                print(f"Registering: new client {id_data} : {c_host, c_pot}")
                if id_data not in self.routes_client:
                    await self.add_mini_client(id_data, (c_host, c_pot))
                    self.routes[str((c_host, c_pot))] = id_data

            # print("self.routes:", self.routes)
        except Exception as e:
            print("Error in on_register", str(e))

    def on_client_exit(self, id_):

        if isinstance(id_, str):
            id_ = eval(id_)

        c_name = self.routes.get(id_)

        if c_name is None:
            return

        if c_name in self.routes_client:
            self.remove_c_route(c_name)
            print(f"Removed route to {c_name}")

    async def add_server_route(self, source_id, addr=None):
        if addr is None:
            addr = ('0.0.0.0', 6588)
        try:
            self.routers_servers[source_id] = await DaemonRout(rout=self.crate_rout(source_id, addr=addr),
                                                               name=source_id,
                                                               on_r=self.on_register)
            self.routers_servers_tasks.append(self.routers_servers[source_id].online)
        except Exception as e:
            print(f"Sever already Online : {e}")

        if not self.routers_servers_tasks_running_flag:
            self.routers_servers_tasks_running_flag = True
            threading.Thread(target=self.server_route_runner, daemon=True).start()

    def server_route_runner(self):
        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)

        # Sammle alle Ergebnisse zusammen
        results = loop.run_until_complete(asyncio.gather(*self.routers_servers_tasks))

        for result in results:
            print(result)

        loop.close()
        self.routers_servers_tasks_running_flag = False

    async def add_js_route(self, source_id="js:web"):
        await self.add_server_route(source_id, ("./web/scripts/tb_socket.sock", 0))

    async def register_event(self, event: Event):

        if event in self.events:
            return Result.default_user_error("Event registration failed Event already registered")

        print(f"Registration new Event : {event.name}, {str(event.event_id)}")
        self.events.add(event)

        if event.scope.name == Scope.instance.name:
            return

        if event.scope.name == Scope.local.name:
            if not self.bo and "P0" not in self.routes_client and os.getenv("TOOLBOXV2_BASE_HOST",
                                                                            "localhost") != "localhost":
                await self.add_client_route("P0", (os.getenv("TOOLBOXV2_BASE_HOST", "localhost"),
                                                   os.getenv("TOOLBOXV2_BASE_PORT", 6568)))
                self.bo = True
            return

        if event.scope.name == Scope.local_network.name:
            if self.identification == "P0" and not self.bo:
                t0 = threading.Thread(target=self.start_brodcast_router_local_network, daemon=True)
                t0.start()
            elif not self.bo and "P0" not in self.routes_client and os.getenv("TOOLBOXV2_BASE_HOST",
                                                                              "localhost") == "localhost":
                self.bo = True
                # self.add_server_route(self.identification, ("127.0.0.1", 44667))
                with Spinner(message="Sercheing for Rooter instance", count_down=True, time_in_s=6):
                    with ThreadPoolExecutor(max_workers=1) as executor:
                        t0 = executor.submit(make_known, self.identification)
                        try:
                            data = t0.result(timeout=6)
                        except TimeoutError:
                            print("No P0 found in network or on device")
                            return
                    print(f"Found P0 on {type(data)} {data.get('host')}")
                    await self.add_client_route("P0", (data.get("host"), os.getenv("TOOLBOXV2_BASE_PORT", 6568)))
            elif not self.bo and "P0" not in self.routes_client and os.getenv("TOOLBOXV2_BASE_HOST",
                                                                              "localhost") != "localhost":
                do = await self.add_client_route("P0", (
                    os.getenv("TOOLBOXV2_BASE_HOST", "localhost"), os.getenv("TOOLBOXV2_BASE_PORT", 6568)))
                self.bo = do
                if not do:
                    print("Connection failed")
                    os.environ["TOOLBOXV2_BASE_HOST"] = "localhost"

        if event.scope.name == Scope.global_network.name:
            await self.add_server_route(self.source_id, ('0.0.0.0', os.getenv("TOOLBOXV2_REMOTE_PORT", 6587)))

    async def connect_to_remote(self, host=os.getenv("TOOLBOXV2_REMOTE_IP"),
                                port=os.getenv("TOOLBOXV2_REMOTE_PORT", 6587)):
        await self.add_client_route("S0", (host, port))

    def start_brodcast_router_local_network(self):
        self.bo = True

        # print("Starting brodcast router 0")
        router = start_client(get_local_ip())
        # print("Starting brodcast router 1")
        # next(router)
        # print("Starting brodcast router")
        while self.running:
            source_id, connection = next(router)
            print(f"Infos :{source_id}, connection :{connection}")
            self.routes[source_id] = connection[0]
            router.send(self.running)

        router.send("e")
        router.close()

    def _get_event_by_id_or_name(self, event_id: str or EventID):
        if isinstance(event_id, str):
            events = [e for e in self.events if e.name == event_id]
            if len(events) < 1:
                return Result.default_user_error("Event not registered")
            event = events[0]

        elif isinstance(event_id, EventID):
            events = [e for e in self.events if e.event_id.ID == event_id.ID]
            if len(events) < 1:
                events = [e for e in self.events if e.name == event_id.ID]
            if len(events) < 1:
                return Result.default_user_error("Event not registered")
            event = events[0]

        elif isinstance(event_id, Event):
            if event_id not in self.events:
                return Result.default_user_error("Event not registered")
            event = event_id

        else:
            event = Result.default_user_error("Event not registered")

        return event

    def remove_event(self, event: Event or EventID or str):

        event = self._get_event_by_id_or_name(event)
        if isinstance(event, Event):
            self.events.remove(event)
        else:
            return event

    async def _trigger_local(self, event_id: EventID):
        """
        Exec source based on

        source_types
            F -> call directly
            R -> use get_app(str(event_id)).run_any(*args, **kwargs)
            S -> evaluate string
        scope
            instance -> _trigger_local
            local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True)
            local_network -> use proxy0 app to communicate withe Daemon0 then local
            global_network ->
        exec_in
        event_id
        threaded

                       """
        event = self._get_event_by_id_or_name(event_id)

        if isinstance(event, Result):
            event.print()
            if self.identification == "P0":
                return event
            print(f"Routing to P0 {self.events}")
            if self.source_id not in self.routes_client:
                # self.routers[self.source_id] = DaemonRout(rout=self.crate_rout(self.source_id))
                await self.add_client_route("P0", ('127.0.0.1', 6568))
            return await self.route_event_id(event_id)

        # if event.threaded:
        #    threading.Thread(target=self.runner, args=(event, event_id), daemon=True).start()
        #    return "Event running In Thread"
        # else:

        return await self.runner(event, event_id)

    async def runner(self, event, event_id: EventID):

        if event.kwargs_ is None:
            event.kwargs_ = {}
        if event.args is None:
            event.args = []

        if event.source_types.name is SourceTypes.P.name:
            return event.source(*event.args, payload=event_id, **event.kwargs_)

        if event.source_types.name is SourceTypes.F.name:
            return event.source(*event.args, **event.kwargs_)

        if event.source_types.name is SourceTypes.R.name:
            return get_app(str(event_id)).run_any(mod_function_name=event.source, get_results=True, args_=event.args,
                                                  kwargs_=event.kwargs_)

        if event.source_types.name is SourceTypes.AP.name:
            if 'payload' in event.kwargs_:
                if event_id.payload != event.kwargs_['payload']:
                    event_id.payload = event.kwargs_['payload']
                del event.kwargs_['payload']
            print(event.args, event.kwargs_, "TODO: remove")
            return await event.source(*event.args, payload=event_id, **event.kwargs_)

        if event.source_types.name is SourceTypes.AF.name:
            return await event.source(*event.args, **event.kwargs_)

        if event.source_types.name is SourceTypes.AR.name:
            return await get_app(str(event_id)).run_any(mod_function_name=event.source, get_results=True,
                                                        args_=event.args,
                                                        kwargs_=event.kwargs_)

        if event.source_types.name is SourceTypes.S.name:
            return eval(event.source, __locals={'app': get_app(str(event_id)), 'event': event, 'eventManagerC': self})

    async def routing_function_router(self, event_id: EventID):

        result = await self.trigger_event(event_id)

        if result is None:
            result = Result.default_user_error("Invalid Event ID")

        if isinstance(result, bytes | dict):
            pass
        elif isinstance(result, Result):
            result.result.data_info = str(event_id)
        elif isinstance(result, EventID):
            result = Result.default_internal_error("Event not found", data=result)
        else:
            result = Result.ok(data=result, data_info="<automatic>", info=str(event_id.path))

        if isinstance(result, str):
            result = result.encode()

        return result

    async def trigger_evnet_by_name(self, name: str):
        await self.trigger_event(EventID.crate_name_as_id(name=name))

    async def trigger_event(self, event_id: EventID):
        """
        Exec source based on

        source_types
            F -> call directly
            R -> use get_app(str(event_id)).run_any(*args, **kwargs)
            S -> evaluate string
        scope
            instance -> _trigger_local
            local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True)
            local_network -> use proxy0 app to communicate withe Daemon0 then local
            global_network ->
        exec_in
        event_id
        threaded

                       """
        # print(f"event-id Ptah : {event_id.get_path()}")
        # print(f"testing trigger_event for {event_id.get_source()} {event_id.get_source()[-1] == self.source_id} ")
        print(str(event_id))
        if event_id.get_source()[-1] == self.source_id:
            payload = await self._trigger_local(event_id)
            event_id.set_payload(payload)
            if len(event_id.path) > 1:
                event_id.source = ':'.join([e.split(':')[0] for e in event_id.get_path() if e != "E"])
                res = await self.route_event_id(event_id)
                if isinstance(res, Result):
                    res.print()
                else:
                    print(res)
            return payload
        return await self.route_event_id(event_id)

    async def route_event_id(self, event_id: EventID):

        # print(f"testing route_event_id for {event_id.get_source()[-1]}")
        if event_id.get_source()[-1] == '*':  # self.identification == "P0" and
            responses = []
            event_id.source = ':'.join(event_id.get_source()[:-1])
            event_id.add_path(f"{self._name}({self.source_id})")
            data = asdict(event_id)
            for name, rout_ in self.routes_client.items():
                if name in event_id.path:
                    continue
                ret = await rout_.put_data(data)
                responses.append(ret)
            return responses
        route = self.routes_client.get(event_id.get_source()[-1])
        # print("route:", route)
        if route is None:
            route = self.routes_client.get(event_id.get_path()[-1])
        if route is None:
            return event_id.add_path(("" if len(event_id.get_source()) == 1 else "404#")+self.identification)
        time.sleep(0.25)
        event_id.source = ':'.join(event_id.get_source()[:-1])
        event_id.add_path(f"{self._name}({self.source_id})")
        return await route.put_data(asdict(event_id))

    async def receiver(self):

        t0 = time.time()

        while self.running:
            time.sleep(0.25)
            if not self.receiver_que.empty():
                event_id = self.receiver_que.get()
                print("Receiver Event", str(event_id))
                await self.trigger_event(event_id)

            if time.time() - t0 > 5:
                await self.receive_all_client_data()
                t0 = time.time()

    def info(self):
        return {"source": self.source_id, "known_routs:": self.routers_servers, "_router": self.routes_client,
                "events": self.events}

    def stop(self):
        self.running = False
        list(map(lambda x: x.disconnect(), self.routes_client.values()))
        list(map(lambda x: x.stop(), self.routers_servers.values()))

    def reconnect(self, name):
        if name is None:
            pass
        elif name in self.routes_client:
            self.routes_client[name].reconnect()
            return
        list(map(lambda x: x.reconnect(), self.routes_client.values()))

    async def verify(self, name):
        if name is None:
            pass
        elif name in self.routes_client:
            await self.routes_client[name].verify()
            return
        for x in self.routes_client.values():
            await x.verify()
trigger_event(event_id) async

Exec source based on

source_types F -> call directly R -> use get_app(str(event_id)).run_any(args, *kwargs) S -> evaluate string scope instance -> _trigger_local local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True) local_network -> use proxy0 app to communicate withe Daemon0 then local global_network -> exec_in event_id threaded

Source code in toolboxv2/mods/EventManager/module.py
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
async def trigger_event(self, event_id: EventID):
    """
    Exec source based on

    source_types
        F -> call directly
        R -> use get_app(str(event_id)).run_any(*args, **kwargs)
        S -> evaluate string
    scope
        instance -> _trigger_local
        local -> if you ar proxy app run the event through get_app(str(event_id)).run_any(TBEF.EventManager._trigger_local, args=args, kwargs=kwargs, get_result=True)
        local_network -> use proxy0 app to communicate withe Daemon0 then local
        global_network ->
    exec_in
    event_id
    threaded

                   """
    # print(f"event-id Ptah : {event_id.get_path()}")
    # print(f"testing trigger_event for {event_id.get_source()} {event_id.get_source()[-1] == self.source_id} ")
    print(str(event_id))
    if event_id.get_source()[-1] == self.source_id:
        payload = await self._trigger_local(event_id)
        event_id.set_payload(payload)
        if len(event_id.path) > 1:
            event_id.source = ':'.join([e.split(':')[0] for e in event_id.get_path() if e != "E"])
            res = await self.route_event_id(event_id)
            if isinstance(res, Result):
                res.print()
            else:
                print(res)
        return payload
    return await self.route_event_id(event_id)
Rout dataclass
Source code in toolboxv2/mods/EventManager/module.py
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
@dataclass
class Rout:
    _from: str
    _to: str

    _from_port: int
    _from_host: str

    _to_port: int
    _to_host: str

    routing_function: Callable

    @property
    def to_host(self):
        return self._to_host

    @property
    def to_port(self):
        return self._to_port

    async def put_data(self, event_id_data: dict[str, str]):
        event_id: EventID = EventID(**event_id_data)
        return await self.routing_function(event_id)

    def close(self):
        """ Close """
close()

Close

Source code in toolboxv2/mods/EventManager/module.py
165
166
def close(self):
    """ Close """

FileWidget

FileUploadHandler

Source code in toolboxv2/mods/FileWidget.py
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
class FileUploadHandler:
    def __init__(self, upload_dir: str = 'uploads'):
        self.upload_dir = Path(upload_dir)
        self.upload_dir.mkdir(parents=True, exist_ok=True)
        # self.app = get_app().app # If logger is needed here

    def save_file(self, chunk_info: ChunkInfo, storage: BlobStorage) -> str:
        """Speichert die Datei oder Chunk. Chunks werden lokal gespeichert, dann zu BlobStorage gemerged."""
        final_blob_path = Path(chunk_info.filename).name  # Use only filename part for security within blob storage

        if chunk_info.total_chunks == 1:
            # Komplette Datei direkt in BlobStorage speichern
            # print(f"Saving single part file: {final_blob_path} to BlobStorage directly.") # Debug
            with BlobFile(final_blob_path, 'w', storage=storage) as bf:
                bf.write(chunk_info.content)
        else:
            # Chunk lokal speichern
            # Sanitize filename for local path (original chunk_info.filename might contain path parts client-side)
            safe_base_filename = "".join(
                c if c.isalnum() or c in ('.', '_', '-') else '_' for c in Path(chunk_info.filename).name)
            chunk_path = self.upload_dir / f"{safe_base_filename}.part{chunk_info.chunk_index}"
            # print(f"Saving chunk: {chunk_path} locally. Total chunks: {chunk_info.total_chunks}") # Debug

            with open(chunk_path, 'wb') as f:
                f.write(chunk_info.content)

            if self._all_chunks_received(safe_base_filename, chunk_info.total_chunks):
                # print(f"All chunks received for {safe_base_filename}. Merging to BlobStorage path: {final_blob_path}") # Debug
                self._merge_chunks_to_blob(safe_base_filename, chunk_info.total_chunks, final_blob_path, storage)
                self._cleanup_chunks(safe_base_filename, chunk_info.total_chunks)
            # else:
            # print(f"Still waiting for more chunks for {safe_base_filename}.") # Debug

        return final_blob_path  # Path within BlobStorage

    def _all_chunks_received(self, safe_base_filename: str, total_chunks: int) -> bool:
        for i in range(total_chunks):
            chunk_path = self.upload_dir / f"{safe_base_filename}.part{i}"
            if not chunk_path.exists():
                # print(f"Chunk {i} for {safe_base_filename} not found. Path: {chunk_path}") # Debug
                return False
        # print(f"All {total_chunks} chunks found for {safe_base_filename}.") # Debug
        return True

    def _merge_chunks_to_blob(self, safe_base_filename: str, total_chunks: int, final_blob_path: str,
                              storage: BlobStorage):
        # print(f"Merging {total_chunks} chunks for {safe_base_filename} into Blob: {final_blob_path}") # Debug
        with BlobFile(final_blob_path, 'w', storage=storage) as outfile:
            for i in range(total_chunks):
                chunk_path = self.upload_dir / f"{safe_base_filename}.part{i}"
                # print(f"Appending chunk {i} ({chunk_path}) to Blob.") # Debug
                with open(chunk_path, 'rb') as chunk_file:
                    outfile.write(chunk_file.read())
        # print(f"Finished merging chunks for {safe_base_filename} to Blob: {final_blob_path}") # Debug

    def _cleanup_chunks(self, safe_base_filename: str, total_chunks: int):
        # print(f"Cleaning up {total_chunks} chunks for {safe_base_filename}.") # Debug
        for i in range(total_chunks):
            chunk_path = self.upload_dir / f"{safe_base_filename}.part{i}"
            if chunk_path.exists():
                # print(f"Removing chunk: {chunk_path}") # Debug
                try:
                    os.remove(chunk_path)
                except OSError as e:
                    # self.app.logger.error(f"Error removing chunk {chunk_path}: {e}") # If logger available
                    print(f"Error removing chunk {chunk_path}: {e}")
save_file(chunk_info, storage)

Speichert die Datei oder Chunk. Chunks werden lokal gespeichert, dann zu BlobStorage gemerged.

Source code in toolboxv2/mods/FileWidget.py
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
def save_file(self, chunk_info: ChunkInfo, storage: BlobStorage) -> str:
    """Speichert die Datei oder Chunk. Chunks werden lokal gespeichert, dann zu BlobStorage gemerged."""
    final_blob_path = Path(chunk_info.filename).name  # Use only filename part for security within blob storage

    if chunk_info.total_chunks == 1:
        # Komplette Datei direkt in BlobStorage speichern
        # print(f"Saving single part file: {final_blob_path} to BlobStorage directly.") # Debug
        with BlobFile(final_blob_path, 'w', storage=storage) as bf:
            bf.write(chunk_info.content)
    else:
        # Chunk lokal speichern
        # Sanitize filename for local path (original chunk_info.filename might contain path parts client-side)
        safe_base_filename = "".join(
            c if c.isalnum() or c in ('.', '_', '-') else '_' for c in Path(chunk_info.filename).name)
        chunk_path = self.upload_dir / f"{safe_base_filename}.part{chunk_info.chunk_index}"
        # print(f"Saving chunk: {chunk_path} locally. Total chunks: {chunk_info.total_chunks}") # Debug

        with open(chunk_path, 'wb') as f:
            f.write(chunk_info.content)

        if self._all_chunks_received(safe_base_filename, chunk_info.total_chunks):
            # print(f"All chunks received for {safe_base_filename}. Merging to BlobStorage path: {final_blob_path}") # Debug
            self._merge_chunks_to_blob(safe_base_filename, chunk_info.total_chunks, final_blob_path, storage)
            self._cleanup_chunks(safe_base_filename, chunk_info.total_chunks)
        # else:
        # print(f"Still waiting for more chunks for {safe_base_filename}.") # Debug

    return final_blob_path  # Path within BlobStorage

access_shared_file(self, request, share_id, filename=None, row=None) async

Accesses a shared file via its share_id. The URL for this would be like /api/FileWidget/shared/{share_id_value} The 'share_id: str' in signature implies ToolBoxV2 extracts it from path.

Source code in toolboxv2/mods/FileWidget.py
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
@export(mod_name=MOD_NAME, api=True, version=VERSION, name="open_shared", api_methods=['GET'],
        request_as_kwarg=True, level=-1, row=True)
async def access_shared_file(self, request: RequestData, share_id: str, filename: str = None, row=None) -> Result:  # share_id from query params
    """
    Accesses a shared file via its share_id.
    The URL for this would be like /api/FileWidget/shared/{share_id_value}
    The 'share_id: str' in signature implies ToolBoxV2 extracts it from path.
    """
    if not share_id:
        return Result.html(data="Share ID is missing in path.", status=302)

    share_info = self.shares.get(share_id) if self.shares is not None else None
    if not share_info:
        return Result.html(data="Share link is invalid or has expired.", status=404)

    owner_uid = share_info["owner_uid"]
    file_path_in_owner_storage = share_info["file_path"]

    try:
        # Get BlobStorage for the owner, not the current request's user (if any)
        owner_storage = await self.get_blob_storage(
            owner_uid_override=owner_uid)  # Crucially, pass request=None if not needed
        self.app.logger.info(
            f"Accessing shared file via link {share_id}: owner {owner_uid}, path {file_path_in_owner_storage}")
        result = await _prepare_file_response(self, owner_storage, file_path_in_owner_storage, row=row is not None)
        if result.is_error():
            self.app.logger.error(f"Error preparing shared file response for {share_id}: {result.info.help_text}")
            return Result.html(data=f"Failed to prepare shared file for download. {result.info.help_text} {result.result.data_info}")
        return result
    except ValueError as e:  # From get_blob_storage if owner_uid is invalid for some reason
        self.app.logger.error(f"Error getting owner's storage for shared file {share_id} (owner {owner_uid}): {e}",
                              exc_info=True)
        return Result.html(data="Could not access owner's storage for shared file.")
    except Exception as e:
        self.app.logger.error(
            f"Error accessing shared file {share_id} (owner {owner_uid}, path {file_path_in_owner_storage}): {e}",
            exc_info=True)
        return Result.html(data="Could not retrieve shared file.")

get_main_ui(self) async

Serves the main HTML UI for the FileWidget.

Source code in toolboxv2/mods/FileWidget.py
598
599
600
601
602
@export(mod_name=MOD_NAME, api=True, version=VERSION, name="ui", api_methods=['GET'])
async def get_main_ui(self) -> Result:
    """Serves the main HTML UI for the FileWidget."""
    html_content = get_template_content()
    return Result.html(data=html_content)

handle_upload(self, request, form_data=None) async

Handles file uploads. Expects chunked data via form_data kwarg from Rust server. 'form_data' structure (from Rust's parsing of multipart) after client sends FormData with fields: 'file' (the blob), 'fileName', 'chunkIndex', 'totalChunks'.

Expected form_data in this Python function: { "file": { // This 'file' key is the NAME of the form field that held the file blob "filename": "original_file_name_for_this_chunk.txt", // from Content-Disposition of the 'file' field part "content_type": "mime/type_of_chunk", "content_base64": "BASE64_ENCODED_CHUNK_CONTENT" }, "fileName": "overall_final_filename.txt", // From a separate form field named 'fileName' "chunkIndex": "0", // From a separate form field named 'chunkIndex' "totalChunks": "5" // From a separate form field named 'totalChunks' }

Source code in toolboxv2/mods/FileWidget.py
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
@export(mod_name=MOD_NAME, api=True, version=VERSION, name="upload", api_methods=['POST'], request_as_kwarg=True)
async def handle_upload(self, request: RequestData, form_data: dict[str, Any] | None = None) -> Result:
    """
    Handles file uploads. Expects chunked data via form_data kwarg from Rust server.
    'form_data' structure (from Rust's parsing of multipart) after client sends FormData with fields:
    'file' (the blob), 'fileName', 'chunkIndex', 'totalChunks'.

    Expected `form_data` in this Python function:
    {
        "file": {  // This 'file' key is the NAME of the form field that held the file blob
            "filename": "original_file_name_for_this_chunk.txt", // from Content-Disposition of the 'file' field part
            "content_type": "mime/type_of_chunk",
            "content_base64": "BASE64_ENCODED_CHUNK_CONTENT"
        },
        "fileName": "overall_final_filename.txt", // From a separate form field named 'fileName'
        "chunkIndex": "0",                        // From a separate form field named 'chunkIndex'
        "totalChunks": "5"                        // From a separate form field named 'totalChunks'
    }
    """
    self.app.logger.debug(
        f"FileWidget: handle_upload called. Received form_data keys: {list(form_data.keys()) if form_data else 'None'}"
    )
    self.app.logger.debug(f"FileWidget: handle_upload called. Received form_data: {request.to_dict()}")
    # self.app.logger.debug(f"Full form_data: {form_data}") # For deeper debugging if needed

    if not form_data:
        return Result.default_user_error(info="No form data received for upload.", exec_code=400)

    try:
        storage = await self.get_blob_storage(request)

        # Extract data from form_data (populated by Rust server from multipart)
        file_field_data = form_data.get('file')  # This is the dict from UploadedFile struct
        # The 'file_field_data.get('filename')' is the name of the chunk part,
        # which the JS client sets to be the same as the original file's name.
        # This is fine for FileUploadHandler.save_file's chunk_info.filename if total_chunks > 1,
        # as it will be used to create temporary part files like "original_file_name.txt.part0".

        overall_filename_from_form = form_data.get('fileName') # This is the target filename for the assembled file.
        chunk_index_str = form_data.get('chunkIndex')
        total_chunks_str = form_data.get('totalChunks')

        if not all([
            file_field_data, isinstance(file_field_data, dict),
            overall_filename_from_form,
            chunk_index_str is not None, # Check for presence, not just truthiness (0 is valid)
            total_chunks_str is not None # Check for presence
        ]):
            missing = []
            if not file_field_data or not isinstance(file_field_data, dict): missing.append("'file' object field")
            if not overall_filename_from_form: missing.append("'fileName' field")
            if chunk_index_str is None: missing.append("'chunkIndex' field")
            if total_chunks_str is None: missing.append("'totalChunks' field")

            self.app.logger.error(
                f"Missing critical form data fields for upload: {missing}. Received form_data: {form_data}")
            return Result.default_user_error(info=f"Incomplete upload data. Missing: {', '.join(missing)}",
                                             exec_code=400)

        content_base64 = file_field_data.get('content_base64')
        if not content_base64:
            return Result.default_user_error(info="File content (base64) not found in 'file' field data.",
                                             exec_code=400)

        try:
            content_bytes = base64.b64decode(content_base64)
        except base64.binascii.Error as b64_error:
            self.app.logger.error(f"Base64 decoding failed for upload: {b64_error}")
            return Result.default_user_error(info="Invalid file content encoding.", exec_code=400)

        try:
            chunk_index = int(chunk_index_str)
            total_chunks = int(total_chunks_str)
        except ValueError:
            return Result.default_user_error(info="Invalid chunk index or total chunks value. Must be integers.", exec_code=400)

        # Use the 'overall_filename_from_form' for the ChunkInfo.filename,
        # as this is the intended final name in blob storage.
        # FileUploadHandler will use Path(this_name).name to ensure it's just a filename.
        chunk_info_to_save = ChunkInfo(
            filename=overall_filename_from_form, # THIS IS THE KEY CHANGE FOR CONSISTENCY
            chunk_index=chunk_index,
            total_chunks=total_chunks,
            content=content_bytes
        )

        self.app.logger.info(
            f"Processing chunk {chunk_index + 1}/{total_chunks} for final file '{overall_filename_from_form}'. " # Log the intended final name
            f"Size: {len(content_bytes)} bytes."
        )

        saved_blob_path = self.upload_handler.save_file(chunk_info_to_save, storage) # saved_blob_path will be Path(overall_filename_from_form).name

        msg = f"Chunk {chunk_index + 1}/{total_chunks} for '{saved_blob_path}' saved."
        if chunk_info_to_save.chunk_index == chunk_info_to_save.total_chunks - 1:
            # Check if fully assembled
            # The 'safe_base_filename' in FileUploadHandler is derived from ChunkInfo.filename,
            # which we've now set to 'overall_filename_from_form'.
            # So, this check should work correctly.
            safe_base_filename_for_check = "".join(
                c if c.isalnum() or c in ('.', '_', '-') else '_' for c in Path(overall_filename_from_form).name)

            # A slight delay might be needed if file system operations are not instantly consistent across threads/processes
            # For now, assume direct check is okay.
            # await asyncio.sleep(0.1) # Optional small delay if race conditions are suspected with file system

            if self.upload_handler._all_chunks_received(safe_base_filename_for_check, total_chunks):
                msg = f"File '{saved_blob_path}' upload complete and assembled."
                self.app.logger.info(msg)
            else:
                msg = f"Final chunk for '{saved_blob_path}' saved, but assembly check failed or is pending."
                self.app.logger.warning(msg + f" (Could not verify all chunks for '{safe_base_filename_for_check}' immediately after final one)")


        return Result.ok(data={"message": msg, "path": saved_blob_path}) # Return the blob-relative path

    except ValueError as e:
        self.app.logger.error(f"Upload processing error: {e}", exc_info=True)
        return Result.default_user_error(info=f"Upload error: {str(e)}",
                                         exec_code=400 if "authentication" in str(e).lower() else 400)
    except Exception as e:
        self.app.logger.error(f"Unexpected error during file upload: {e}", exc_info=True)
        return Result.default_internal_error(info="An unexpected error occurred during upload.")

P2PRPCClient

P2PRPCClient

Source code in toolboxv2/mods/P2PRPCClient.py
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
class P2PRPCClient:
    def __init__(self, app: App, host: str, port: int, tb_r_key: str = None):
        self.app = app
        self.host = host
        self.port = port
        self.reader = None
        self.writer = None
        self.futures = {}
        self.code = Code()

        if tb_r_key is None:
            tb_r_key = os.getenv("TB_R_KEY")
            if tb_r_key is None:
                raise ValueError("TB_R_KEY environment variable is not set.")

        if len(tb_r_key) < 24:
            raise ValueError("TB_R_KEY must be at least 24 characters long for security.")
        self.auth_key_part = tb_r_key[:24]
        self.identification_part = tb_r_key[24:]
        self.session_key = None

    async def connect(self):
        """Connects to the local tcm instance and performs key exchange."""
        try:
            self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
            print(f"RPC Client: Connected to tcm at {self.host}:{self.port}")

            # Receive encrypted session key from server
            len_data = await self.reader.readexactly(4)
            encrypted_session_key_len = int.from_bytes(len_data, 'big')
            encrypted_session_key = (await self.reader.readexactly(encrypted_session_key_len)).decode('utf-8')

            # Decrypt session key using auth_key_part
            self.session_key = self.code.decrypt_symmetric(encrypted_session_key, self.auth_key_part)

            # Send challenge back to server, encrypted with session key
            challenge = "CHALLENGE_ACK"
            encrypted_challenge = self.code.encrypt_symmetric(challenge, self.session_key)
            self.writer.write(len(encrypted_challenge).to_bytes(4, 'big'))
            self.writer.write(encrypted_challenge.encode('utf-8'))
            await self.writer.drain()

            # Start a background task to listen for responses
            asyncio.create_task(self.listen_for_responses())

        except ConnectionRefusedError:
            print(f"RPC Client: Connection to {self.host}:{self.port} refused. Is the tcm peer running?")
            raise
        except Exception as e:
            print(f"RPC Client: Error during connection/key exchange: {e}")
            raise

    async def listen_for_responses(self):
        """Listens for incoming responses, decrypts them, and resolves the corresponding future."""
        try:
            while True:
                len_data = await self.reader.readexactly(4)
                msg_len = int.from_bytes(len_data, 'big')
                encrypted_msg_data = (await self.reader.readexactly(msg_len)).decode('utf-8')

                decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, self.session_key)
                response = json.loads(decrypted_msg_data)

                call_id = response.get('call_id')
                if call_id in self.futures:
                    future = self.futures.pop(call_id)
                    future.set_result(response)
        except asyncio.IncompleteReadError:
            print("RPC Client: Connection closed.")
        except Exception as e:
            print(f"RPC Client: Error listening for responses: {e}")
        finally:
            # Clean up any pending futures
            for future in self.futures.values():
                future.set_exception(ConnectionError("Connection lost"))
            self.futures.clear()

    async def call(self, module: str, function: str, *args, **kwargs):
        """Makes a remote procedure call."""
        if not self.writer:
            await self.connect()

        call_id = str(uuid.uuid4())
        request = {
            "type": "request",
            "call_id": call_id,
            "module": module,
            "function": function,
            "args": args,
            "kwargs": kwargs,
            "identification_part": self.identification_part
        }

        future = asyncio.get_running_loop().create_future()
        self.futures[call_id] = future

        try:
            request_str = json.dumps(request)
            encrypted_request = self.code.encrypt_symmetric(request_str, self.session_key)

            self.writer.write(len(encrypted_request).to_bytes(4, 'big'))
            self.writer.write(encrypted_request.encode('utf-8'))
            await self.writer.drain()

            # Wait for the response with a timeout
            response = await asyncio.wait_for(future, timeout=30.0)

            if response.get('error'):
                return Result(**response['error'])
            else:
                return Result.ok(response.get('result'))

        except TimeoutError:
            self.futures.pop(call_id, None)
            return Result.default_internal_error("RPC call timed out.")
        except Exception as e:
            self.futures.pop(call_id, None)
            return Result.default_internal_error(f"RPC call failed: {e}")

    async def close(self):
        """Closes the connection."""
        if self.writer:
            self.writer.close()
            await self.writer.wait_closed()
            print("RPC Client: Connection closed.")
call(module, function, *args, **kwargs) async

Makes a remote procedure call.

Source code in toolboxv2/mods/P2PRPCClient.py
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
async def call(self, module: str, function: str, *args, **kwargs):
    """Makes a remote procedure call."""
    if not self.writer:
        await self.connect()

    call_id = str(uuid.uuid4())
    request = {
        "type": "request",
        "call_id": call_id,
        "module": module,
        "function": function,
        "args": args,
        "kwargs": kwargs,
        "identification_part": self.identification_part
    }

    future = asyncio.get_running_loop().create_future()
    self.futures[call_id] = future

    try:
        request_str = json.dumps(request)
        encrypted_request = self.code.encrypt_symmetric(request_str, self.session_key)

        self.writer.write(len(encrypted_request).to_bytes(4, 'big'))
        self.writer.write(encrypted_request.encode('utf-8'))
        await self.writer.drain()

        # Wait for the response with a timeout
        response = await asyncio.wait_for(future, timeout=30.0)

        if response.get('error'):
            return Result(**response['error'])
        else:
            return Result.ok(response.get('result'))

    except TimeoutError:
        self.futures.pop(call_id, None)
        return Result.default_internal_error("RPC call timed out.")
    except Exception as e:
        self.futures.pop(call_id, None)
        return Result.default_internal_error(f"RPC call failed: {e}")
close() async

Closes the connection.

Source code in toolboxv2/mods/P2PRPCClient.py
133
134
135
136
137
138
async def close(self):
    """Closes the connection."""
    if self.writer:
        self.writer.close()
        await self.writer.wait_closed()
        print("RPC Client: Connection closed.")
connect() async

Connects to the local tcm instance and performs key exchange.

Source code in toolboxv2/mods/P2PRPCClient.py
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
async def connect(self):
    """Connects to the local tcm instance and performs key exchange."""
    try:
        self.reader, self.writer = await asyncio.open_connection(self.host, self.port)
        print(f"RPC Client: Connected to tcm at {self.host}:{self.port}")

        # Receive encrypted session key from server
        len_data = await self.reader.readexactly(4)
        encrypted_session_key_len = int.from_bytes(len_data, 'big')
        encrypted_session_key = (await self.reader.readexactly(encrypted_session_key_len)).decode('utf-8')

        # Decrypt session key using auth_key_part
        self.session_key = self.code.decrypt_symmetric(encrypted_session_key, self.auth_key_part)

        # Send challenge back to server, encrypted with session key
        challenge = "CHALLENGE_ACK"
        encrypted_challenge = self.code.encrypt_symmetric(challenge, self.session_key)
        self.writer.write(len(encrypted_challenge).to_bytes(4, 'big'))
        self.writer.write(encrypted_challenge.encode('utf-8'))
        await self.writer.drain()

        # Start a background task to listen for responses
        asyncio.create_task(self.listen_for_responses())

    except ConnectionRefusedError:
        print(f"RPC Client: Connection to {self.host}:{self.port} refused. Is the tcm peer running?")
        raise
    except Exception as e:
        print(f"RPC Client: Error during connection/key exchange: {e}")
        raise
listen_for_responses() async

Listens for incoming responses, decrypts them, and resolves the corresponding future.

Source code in toolboxv2/mods/P2PRPCClient.py
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
async def listen_for_responses(self):
    """Listens for incoming responses, decrypts them, and resolves the corresponding future."""
    try:
        while True:
            len_data = await self.reader.readexactly(4)
            msg_len = int.from_bytes(len_data, 'big')
            encrypted_msg_data = (await self.reader.readexactly(msg_len)).decode('utf-8')

            decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, self.session_key)
            response = json.loads(decrypted_msg_data)

            call_id = response.get('call_id')
            if call_id in self.futures:
                future = self.futures.pop(call_id)
                future.set_result(response)
    except asyncio.IncompleteReadError:
        print("RPC Client: Connection closed.")
    except Exception as e:
        print(f"RPC Client: Error listening for responses: {e}")
    finally:
        # Clean up any pending futures
        for future in self.futures.values():
            future.set_exception(ConnectionError("Connection lost"))
        self.futures.clear()

test_rpc_client(app, host='127.0.0.1', port=8000, tb_r_key=None) async

An example of how to use the P2P RPC Client.

Source code in toolboxv2/mods/P2PRPCClient.py
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
@export(mod_name=Name, name="test_rpc_client", test=False)
async def test_rpc_client(app: App, host: str = '127.0.0.1', port: int = 8000, tb_r_key: str = None):
    """An example of how to use the P2P RPC Client."""
    if tb_r_key is None:
        tb_r_key = os.getenv("TB_R_KEY")
        if tb_r_key is None:
            raise ValueError("TB_R_KEY environment variable is not set.")

    client = P2PRPCClient(app, host, port, tb_r_key)
    try:
        await client.connect()
        # Example: Call the 'list-users' function from the 'helper' module
        result = await client.call("helper", "list-users")
        result.print()
    finally:
        await client.close()

P2PRPCServer

P2PRPCServer

Source code in toolboxv2/mods/P2PRPCServer.py
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
class P2PRPCServer:
    def __init__(self, app: App, host: str, port: int, tb_r_key: str, function_access_config: dict = None):
        self.app = app
        self.host = host
        self.port = port
        self.server = None
        self.code = Code()

        if len(tb_r_key) < 24:
            raise ValueError("TB_R_KEY must be at least 24 characters long for security.")
        self.auth_key_part = tb_r_key[:24]
        self.identification_part_server = tb_r_key[24:]

        self.function_access_config = function_access_config if function_access_config is not None else {}

    async def handle_client(self, reader, writer):
        """Callback to handle a single client connection from a tcm instance."""
        addr = writer.get_extra_info('peername')
        print(f"RPC Server: New connection from {addr}")

        session_key = self.code.generate_symmetric_key()
        encrypted_session_key = self.code.encrypt_symmetric(session_key, self.auth_key_part)

        try:
            writer.write(len(encrypted_session_key).to_bytes(4, 'big'))
            writer.write(encrypted_session_key.encode('utf-8'))
            await writer.drain()

            len_data = await reader.readexactly(4)
            encrypted_challenge_len = int.from_bytes(len_data, 'big')
            encrypted_challenge = (await reader.readexactly(encrypted_challenge_len)).decode('utf-8')

            decrypted_challenge = self.code.decrypt_symmetric(encrypted_challenge, session_key)
            if decrypted_challenge != "CHALLENGE_ACK":
                raise ValueError("Invalid challenge received.")

            print(f"RPC Server: Authenticated client {addr}")

            while True:
                len_data = await reader.readexactly(4)
                msg_len = int.from_bytes(len_data, 'big')

                encrypted_msg_data = (await reader.readexactly(msg_len)).decode('utf-8')

                decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, session_key)

                response = await self.process_rpc(decrypted_msg_data, session_key)

                encrypted_response = self.code.encrypt_symmetric(json.dumps(response), session_key)

                writer.write(len(encrypted_response).to_bytes(4, 'big'))
                writer.write(encrypted_response.encode('utf-8'))
                await writer.drain()

        except asyncio.IncompleteReadError:
            print(f"RPC Server: Connection from {addr} closed.")
        except Exception as e:
            print(f"RPC Server: Error with client {addr}: {e}")
        finally:
            writer.close()
            await writer.wait_closed()

    async def process_rpc(self, msg_data: str, session_key: str) -> dict:
        """Processes a single RPC request and returns a response dictionary."""
        try:
            call = json.loads(msg_data)
            if call.get('type') != 'request':
                raise ValueError("Invalid message type")
        except (json.JSONDecodeError, ValueError) as e:
            return self.format_error(call.get('call_id'), -32700, f"Parse error: {e}")

        call_id = call.get('call_id')
        module = call.get('module')
        function = call.get('function')
        args = call.get('args', [])
        kwargs = call.get('kwargs', {})
        client_identification = call.get('identification_part')

        if not self.is_function_allowed(module, function, client_identification):
            error_msg = f"Function '{module}.{function}' is not allowed for identification '{client_identification}'."
            print(f"RPC Server: {error_msg}")
            return self.format_error(call_id, -32601, "Method not found or not allowed")

        print(f"RPC Server: Executing '{module}.{function}' for '{client_identification}'")
        try:
            result: Result = await self.app.a_run_any(
                (module, function),
                args_=args,
                kwargs_=kwargs,
                get_results=True
            )

            if result.is_error():
                return self.format_error(call_id, result.info.get('exec_code', -32000), result.info.get('help_text'), result.get())
            else:
                return {
                    "type": "response",
                    "call_id": call_id,
                    "result": result.get(),
                    "error": None
                }
        except Exception as e:
            print(f"RPC Server: Exception during execution of '{module}.{function}': {e}")
            return self.format_error(call_id, -32603, "Internal error during execution", str(e))

    def is_function_allowed(self, module: str, function: str, client_identification: str) -> bool:
        """Checks if a function is allowed for a given client identification."""
        if module not in self.function_access_config:
            return False

        allowed_functions_for_module = self.function_access_config[module]

        if function not in allowed_functions_for_module:
            return False

        # If the function is whitelisted, and there's a specific identification part,
        # you might want to add more granular control here.
        # For now, if it's in the whitelist, it's allowed for any identified client.
        # You could extend function_access_config to be:
        # {"ModuleName": {"function1": ["id1", "id2"], "function2": ["id3"]}}
        # For simplicity, current implementation assumes if module.function is in whitelist,
        # it's generally allowed for any authenticated client.
        return True

    def format_error(self, call_id, code, message, details=None) -> dict:
        """Helper to create a JSON-RPC error response object."""
        return {
            "type": "response",
            "call_id": call_id,
            "result": None,
            "error": {
                "code": code,
                "message": message,
                "details": details
            }
        }

    async def start(self):
        """Starts the TCP server."""
        self.server = await asyncio.start_server(
            self.handle_client, self.host, self.port
        )
        addr = self.server.sockets[0].getsockname()
        print(f"P2P RPC Server listening on {addr}")
        async with self.server:
            await self.server.serve_forever()

    def stop(self):
        """Stops the TCP server."""
        if self.server:
            self.server.close()
            print("P2P RPC Server stopped.")
format_error(call_id, code, message, details=None)

Helper to create a JSON-RPC error response object.

Source code in toolboxv2/mods/P2PRPCServer.py
137
138
139
140
141
142
143
144
145
146
147
148
def format_error(self, call_id, code, message, details=None) -> dict:
    """Helper to create a JSON-RPC error response object."""
    return {
        "type": "response",
        "call_id": call_id,
        "result": None,
        "error": {
            "code": code,
            "message": message,
            "details": details
        }
    }
handle_client(reader, writer) async

Callback to handle a single client connection from a tcm instance.

Source code in toolboxv2/mods/P2PRPCServer.py
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
async def handle_client(self, reader, writer):
    """Callback to handle a single client connection from a tcm instance."""
    addr = writer.get_extra_info('peername')
    print(f"RPC Server: New connection from {addr}")

    session_key = self.code.generate_symmetric_key()
    encrypted_session_key = self.code.encrypt_symmetric(session_key, self.auth_key_part)

    try:
        writer.write(len(encrypted_session_key).to_bytes(4, 'big'))
        writer.write(encrypted_session_key.encode('utf-8'))
        await writer.drain()

        len_data = await reader.readexactly(4)
        encrypted_challenge_len = int.from_bytes(len_data, 'big')
        encrypted_challenge = (await reader.readexactly(encrypted_challenge_len)).decode('utf-8')

        decrypted_challenge = self.code.decrypt_symmetric(encrypted_challenge, session_key)
        if decrypted_challenge != "CHALLENGE_ACK":
            raise ValueError("Invalid challenge received.")

        print(f"RPC Server: Authenticated client {addr}")

        while True:
            len_data = await reader.readexactly(4)
            msg_len = int.from_bytes(len_data, 'big')

            encrypted_msg_data = (await reader.readexactly(msg_len)).decode('utf-8')

            decrypted_msg_data = self.code.decrypt_symmetric(encrypted_msg_data, session_key)

            response = await self.process_rpc(decrypted_msg_data, session_key)

            encrypted_response = self.code.encrypt_symmetric(json.dumps(response), session_key)

            writer.write(len(encrypted_response).to_bytes(4, 'big'))
            writer.write(encrypted_response.encode('utf-8'))
            await writer.drain()

    except asyncio.IncompleteReadError:
        print(f"RPC Server: Connection from {addr} closed.")
    except Exception as e:
        print(f"RPC Server: Error with client {addr}: {e}")
    finally:
        writer.close()
        await writer.wait_closed()
is_function_allowed(module, function, client_identification)

Checks if a function is allowed for a given client identification.

Source code in toolboxv2/mods/P2PRPCServer.py
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
def is_function_allowed(self, module: str, function: str, client_identification: str) -> bool:
    """Checks if a function is allowed for a given client identification."""
    if module not in self.function_access_config:
        return False

    allowed_functions_for_module = self.function_access_config[module]

    if function not in allowed_functions_for_module:
        return False

    # If the function is whitelisted, and there's a specific identification part,
    # you might want to add more granular control here.
    # For now, if it's in the whitelist, it's allowed for any identified client.
    # You could extend function_access_config to be:
    # {"ModuleName": {"function1": ["id1", "id2"], "function2": ["id3"]}}
    # For simplicity, current implementation assumes if module.function is in whitelist,
    # it's generally allowed for any authenticated client.
    return True
process_rpc(msg_data, session_key) async

Processes a single RPC request and returns a response dictionary.

Source code in toolboxv2/mods/P2PRPCServer.py
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
async def process_rpc(self, msg_data: str, session_key: str) -> dict:
    """Processes a single RPC request and returns a response dictionary."""
    try:
        call = json.loads(msg_data)
        if call.get('type') != 'request':
            raise ValueError("Invalid message type")
    except (json.JSONDecodeError, ValueError) as e:
        return self.format_error(call.get('call_id'), -32700, f"Parse error: {e}")

    call_id = call.get('call_id')
    module = call.get('module')
    function = call.get('function')
    args = call.get('args', [])
    kwargs = call.get('kwargs', {})
    client_identification = call.get('identification_part')

    if not self.is_function_allowed(module, function, client_identification):
        error_msg = f"Function '{module}.{function}' is not allowed for identification '{client_identification}'."
        print(f"RPC Server: {error_msg}")
        return self.format_error(call_id, -32601, "Method not found or not allowed")

    print(f"RPC Server: Executing '{module}.{function}' for '{client_identification}'")
    try:
        result: Result = await self.app.a_run_any(
            (module, function),
            args_=args,
            kwargs_=kwargs,
            get_results=True
        )

        if result.is_error():
            return self.format_error(call_id, result.info.get('exec_code', -32000), result.info.get('help_text'), result.get())
        else:
            return {
                "type": "response",
                "call_id": call_id,
                "result": result.get(),
                "error": None
            }
    except Exception as e:
        print(f"RPC Server: Exception during execution of '{module}.{function}': {e}")
        return self.format_error(call_id, -32603, "Internal error during execution", str(e))
start() async

Starts the TCP server.

Source code in toolboxv2/mods/P2PRPCServer.py
150
151
152
153
154
155
156
157
158
async def start(self):
    """Starts the TCP server."""
    self.server = await asyncio.start_server(
        self.handle_client, self.host, self.port
    )
    addr = self.server.sockets[0].getsockname()
    print(f"P2P RPC Server listening on {addr}")
    async with self.server:
        await self.server.serve_forever()
stop()

Stops the TCP server.

Source code in toolboxv2/mods/P2PRPCServer.py
160
161
162
163
164
def stop(self):
    """Stops the TCP server."""
    if self.server:
        self.server.close()
        print("P2P RPC Server stopped.")

start_rpc_server(app, host='127.0.0.1', port=8888, tb_r_key=None, function_access_config=None) async

Starts the P2P RPC server.

Source code in toolboxv2/mods/P2PRPCServer.py
166
167
168
169
170
171
172
173
174
175
176
177
178
@export(mod_name=Name, name="start_server", test=False)
async def start_rpc_server(app: App, host: str = '127.0.0.1', port: int = 8888, tb_r_key: str = None, function_access_config: dict = None):
    """Starts the P2P RPC server."""
    if tb_r_key is None:
        tb_r_key = os.getenv("TB_R_KEY")
        if tb_r_key is None:
            raise ValueError("TB_R_KEY environment variable is not set.")

    server = P2PRPCServer(app, host, port, tb_r_key, function_access_config)
    try:
        await server.start()
    except KeyboardInterrupt:
        server.stop()

POA

module

ActionManagerEnhanced
Source code in toolboxv2/mods/POA/module.py
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
class ActionManagerEnhanced:
    DB_ITEMS_PREFIX = "donext_items"
    DB_HISTORY_PREFIX = "donext_history"
    DB_CURRENT_ITEM_PREFIX = "donext_current_item"
    DB_UNDO_LOG_PREFIX = "donext_undo_log"
    DB_SETTINGS_PREFIX = "donext_settings"  # Added for user settings

    def __init__(self, app: App, user_id: str):
        self.app = app
        self.user_id = user_id
        self.db = app.get_mod("DB")
        self.isaa = app.get_mod("isaa")

        self.settings: UserSettings = UserSettings(user_id=user_id)  # Initialize with defaults
        self.items: list[ActionItem] = []
        self.history: list[HistoryEntry] = []
        self.current_item: ActionItem | None = None
        self.undo_log: list[UndoLogEntry] = []

        self._load_settings()  # Load settings first as they might affect item loading
        self._load_data()

    def _get_db_key(self, prefix: str) -> str:
        return f"{prefix}_{self.user_id}"

    def get_user_timezone(self) -> pytz.BaseTzInfo:
        try:
            return pytz.timezone(self.settings.timezone)
        except pytz.UnknownTimeZoneError:
            return pytz.utc

    def _load_settings(self):
        settings_key = self._get_db_key(self.DB_SETTINGS_PREFIX)
        try:
            settings_data = self.db.get(settings_key)
            if settings_data.is_data() and settings_data.get():
                loaded_settings = json.loads(settings_data.get()[0]) if isinstance(settings_data.get(),
                                                                                   list) else json.loads(
                    settings_data.get())
                self.settings = UserSettings.model_validate_json_safe(loaded_settings)
            else:  # Save default settings if not found
                self._save_settings()
        except Exception as e:
            self.app.logger.error(f"Error loading settings for user {self.user_id}: {e}. Using defaults.")
            self.settings = UserSettings(user_id=self.user_id)  # Fallback to defaults
            self._save_settings()  # Attempt to save defaults

    def _save_settings(self):
        try:
            self.db.set(self._get_db_key(self.DB_SETTINGS_PREFIX), json.dumps(self.settings.model_dump_json_safe()))
        except Exception as e:
            self.app.logger.error(f"Error saving settings for user {self.user_id}: {e}")

    def update_user_settings(self, settings_data: dict[str, Any]) -> UserSettings:
        # Ensure user_id is not changed by malicious input
        current_user_id = self.settings.user_id
        updated_settings = UserSettings.model_validate(
            {**self.settings.model_dump(), **settings_data, "user_id": current_user_id})
        self.settings = updated_settings
        self._save_settings()
        # Potentially re-process items if timezone change affects interpretations, though this is complex.
        # For now, new items will use the new timezone. Existing UTC times remain.
        self.app.logger.info(f"User {self.user_id} settings updated: Timezone {self.settings.timezone}")
        return self.settings

    def _load_data(self):
        items_key = self._get_db_key(self.DB_ITEMS_PREFIX)
        history_key = self._get_db_key(self.DB_HISTORY_PREFIX)
        current_item_key = self._get_db_key(self.DB_CURRENT_ITEM_PREFIX)
        undo_log_key = self._get_db_key(self.DB_UNDO_LOG_PREFIX)
        user_tz_str = self.settings.timezone  # For model_validate_json_safe context

        try:
            items_data = self.db.get(items_key)
            if items_data.is_data() and items_data.get():
                loaded_items_raw = json.loads(items_data.get()[0]) if isinstance(items_data.get(),
                                                                                 list) else json.loads(items_data.get())
                self.items = [ActionItem.model_validate_json_safe(item_dict, user_timezone_str=user_tz_str) for
                              item_dict in loaded_items_raw]

            history_data = self.db.get(history_key)
            if history_data.is_data() and history_data.get():
                loaded_history_raw = json.loads(history_data.get()[0]) if isinstance(history_data.get(),
                                                                                     list) else json.loads(
                    history_data.get())
                self.history = [HistoryEntry.model_validate_json_safe(entry_dict) for entry_dict in loaded_history_raw]

            current_item_data = self.db.get(current_item_key)
            if current_item_data.is_data() and current_item_data.get():
                current_item_dict = json.loads(current_item_data.get()[0]) if isinstance(current_item_data.get(),
                                                                                         list) else json.loads(
                    current_item_data.get())
                if current_item_dict:
                    self.current_item = ActionItem.model_validate_json_safe(current_item_dict,
                                                                            user_timezone_str=user_tz_str)

            undo_log_data = self.db.get(undo_log_key)
            if undo_log_data.is_data() and undo_log_data.get():
                loaded_undo_raw = json.loads(undo_log_data.get()[0]) if isinstance(undo_log_data.get(),
                                                                                   list) else json.loads(
                    undo_log_data.get())
                self.undo_log = [UndoLogEntry.model_validate_json_safe(entry_dict) for entry_dict in loaded_undo_raw]

        except Exception as e:
            self.app.logger.error(f"Error loading data for user {self.user_id}: {e}")
            self.items, self.history, self.current_item, self.undo_log = [], [], None, []
        self._recalculate_next_due_for_all()

    def _save_data(self):
        try:
            self.db.set(self._get_db_key(self.DB_ITEMS_PREFIX),
                        json.dumps([item.model_dump_json_safe() for item in self.items]))
            self.db.set(self._get_db_key(self.DB_HISTORY_PREFIX),
                        json.dumps([entry.model_dump_json_safe() for entry in self.history]))
            self.db.set(self._get_db_key(self.DB_CURRENT_ITEM_PREFIX),
                        json.dumps(self.current_item.model_dump_json_safe() if self.current_item else None))
            self.db.set(self._get_db_key(self.DB_UNDO_LOG_PREFIX),
                        json.dumps([entry.model_dump_json_safe() for entry in self.undo_log]))
        except Exception as e:
            self.app.logger.error(f"Error saving data for user {self.user_id}: {e}")

    def _add_history_entry(self, item: ActionItem, status_override: ActionStatus | None = None,
                           notes: str | None = None):
        entry = HistoryEntry(
            item_id=item.id, item_title=item.title, item_type=item.item_type,
            status_changed_to=status_override or item.status,
            parent_id=item.parent_id, notes=notes
        )
        self.history.append(entry)

    def _datetime_to_user_tz(self, dt_utc: datetime | None) -> datetime | None:
        if not dt_utc: return None
        if dt_utc.tzinfo is None: dt_utc = pytz.utc.localize(dt_utc)  # Should already be UTC
        return dt_utc.astimezone(self.get_user_timezone())

    def _datetime_from_user_input_str(self, dt_str: str | None) -> datetime | None:
        if not dt_str: return None
        try:
            dt = isoparse(dt_str)
            if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:  # Naive
                return self.get_user_timezone().localize(dt).astimezone(pytz.utc)
            return dt.astimezone(pytz.utc)  # Aware, convert to UTC
        except ValueError:
            self.app.logger.warning(f"Could not parse datetime string: {dt_str}")
            return None

    def _recalculate_next_due(self, item: ActionItem):
        now_utc = datetime.now(pytz.utc)
        user_tz = self.get_user_timezone()

        if item.status == ActionStatus.COMPLETED and item.item_type == ItemType.TASK:
            if item.frequency and item.frequency != Frequency.ONE_TIME:
                base_time_utc = item.last_completed or now_utc  # last_completed is already UTC

                # If item had a fixed_time, align next_due to that time of day in user's timezone
                if item.fixed_time:
                    original_fixed_time_user_tz = item.fixed_time.astimezone(user_tz)
                    # Start from last_completed (or now if missing) in user's timezone for calculation
                    base_time_user_tz = base_time_utc.astimezone(user_tz)

                    # Ensure base_time_user_tz is at least original_fixed_time_user_tz for alignment
                    # but calculations should project from last completion.
                    # For example, if daily task due 9am was completed at 11am, next one is tomorrow 9am.
                    # If completed at 8am, next one is today 9am (if fixed_time was today 9am) or tomorrow 9am.

                    # Let's use last_completed as the primary anchor for when the *next* cycle starts.
                    # The original fixed_time's time component is used for the *time of day* of the next due.

                    current_anchor_user_tz = base_time_user_tz

                    # Calculate next occurrence based on frequency
                    if item.frequency == Frequency.DAILY:
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(days=1)).date()
                    elif item.frequency == Frequency.WEEKLY:
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(weeks=1)).date()
                    elif item.frequency == Frequency.MONTHLY:  # Simplified
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(days=30)).date()
                    elif item.frequency == Frequency.ANNUALLY:
                        next_due_user_tz_date = (current_anchor_user_tz + timedelta(days=365)).date()
                    else:  # Should not happen for recurring
                        item.next_due = None
                        return

                    # Combine with original time of day
                    next_due_user_tz = datetime.combine(next_due_user_tz_date, original_fixed_time_user_tz.time(),
                                                        tzinfo=user_tz)
                    item.next_due = next_due_user_tz.astimezone(pytz.utc)

                else:  # No original fixed_time, so recur based on current time of completion
                    if item.frequency == Frequency.DAILY:
                        item.next_due = base_time_utc + timedelta(days=1)
                    elif item.frequency == Frequency.WEEKLY:
                        item.next_due = base_time_utc + timedelta(weeks=1)
                    elif item.frequency == Frequency.MONTHLY:
                        item.next_due = base_time_utc + timedelta(days=30)
                    elif item.frequency == Frequency.ANNUALLY:
                        item.next_due = base_time_utc + timedelta(days=365)

                # Advance until future if needed (e.g., completing an overdue recurring task)
                # This loop must operate on user's local time perception of "next day"
                while item.next_due and item.next_due < now_utc:
                    next_due_user = item.next_due.astimezone(user_tz)
                    original_time_comp = next_due_user.time()  # Preserve time of day

                    if item.frequency == Frequency.DAILY:
                        next_due_user_adv = next_due_user + timedelta(days=1)
                    elif item.frequency == Frequency.WEEKLY:
                        next_due_user_adv = next_due_user + timedelta(weeks=1)
                    # For monthly/annually, simple timedelta might shift day of month. Using replace for date part.
                    elif item.frequency == Frequency.MONTHLY:
                        # This simplified logic might need dateutil.relativedelta for accuracy
                        year, month = (next_due_user.year, next_due_user.month + 1) if next_due_user.month < 12 else (
                            next_due_user.year + 1, 1)
                        try:
                            next_due_user_adv = next_due_user.replace(year=year, month=month)
                        except ValueError:  # Handle e.g. trying to set Feb 30
                            import calendar
                            last_day = calendar.monthrange(year, month)[1]
                            next_due_user_adv = next_due_user.replace(year=year, month=month, day=last_day)

                    elif item.frequency == Frequency.ANNUALLY:
                        try:
                            next_due_user_adv = next_due_user.replace(year=next_due_user.year + 1)
                        except ValueError:  # Handle leap day if original was Feb 29
                            next_due_user_adv = next_due_user.replace(year=next_due_user.year + 1,
                                                                      day=28)  # Or March 1st
                    else:
                        break

                    item.next_due = user_tz.localize(
                        datetime.combine(next_due_user_adv.date(), original_time_comp)).astimezone(pytz.utc)

                item.status = ActionStatus.NOT_STARTED  # Reset for next occurrence
            else:  # One-time task
                item.next_due = None
        elif item.status == ActionStatus.NOT_STARTED and item.fixed_time and not item.next_due:
            item.next_due = item.fixed_time  # fixed_time is already UTC

        # If task is not completed, not started, and has a next_due in the past, but also a fixed_time in the future
        # (e.g. recurring task whose current instance was missed, but fixed_time points to a specific time for all instances)
        # ensure next_due is not before fixed_time if fixed_time is relevant for setting.
        # This logic is complex. Current setup: fixed_time is the "template", next_due is the "instance".

    def _recalculate_next_due_for_all(self):
        for item in self.items:
            self._recalculate_next_due(item)

    def add_item(self, item_data: dict[str, Any], by_ai: bool = False, imported: bool = False) -> ActionItem:
        item_data['_user_timezone_str'] = self.settings.timezone  # For validation context
        item = ActionItem.model_validate(
            item_data)  # Pydantic handles string->datetime, then model_validator converts to UTC
        item.created_by_ai = by_ai
        item.updated_at = datetime.now(pytz.utc)  # Ensure update

        # Initial next_due for new items if not already set by iCal import logic
        if not item.next_due and item.fixed_time and item.status == ActionStatus.NOT_STARTED:
            item.next_due = item.fixed_time

        self.items.append(item)
        self._add_history_entry(item, status_override=ActionStatus.NOT_STARTED,
                                notes="Item created" + (" by AI" if by_ai else "") + (
                                    " via import" if imported else ""))
        if by_ai:
            self._log_ai_action("ai_create_item", [item.id])

        self._save_data()
        return item

    def get_item_by_id(self, item_id: str) -> ActionItem | None:
        return next((item for item in self.items if item.id == item_id), None)

    def update_item(self, item_id: str, update_data: dict[str, Any], by_ai: bool = False) -> ActionItem | None:
        item = self.get_item_by_id(item_id)
        if not item: return None

        previous_data_json = item.model_dump_json() if by_ai else None

        # Pass user timezone for validation context if datetime strings are present
        update_data_with_tz_context = {**update_data, '_user_timezone_str': self.settings.timezone}

        updated_item_dict = item.model_dump()
        updated_item_dict.update(update_data_with_tz_context)

        try:
            # Re-validate the whole model to ensure consistency and proper conversions
            new_item_state = ActionItem.model_validate(updated_item_dict)
            # Preserve original ID and created_at, apply new state
            new_item_state.id = item.id
            new_item_state.created_at = item.created_at
            self.items[self.items.index(item)] = new_item_state
            item = new_item_state
        except Exception as e:
            self.app.logger.error(f"Error validating updated item data: {e}. Update aborted for item {item_id}.")
            return None  # Or raise error

        item.updated_at = datetime.now(pytz.utc)
        item.created_by_ai = by_ai

        self._recalculate_next_due(item)
        self._add_history_entry(item, notes="Item updated" + (" by AI" if by_ai else ""))

        if by_ai:
            self._log_ai_action("ai_modify_item", [item.id],
                                {item.id: previous_data_json} if previous_data_json else None)

        self._save_data()
        return item

    def remove_item(self, item_id: str, record_history: bool = True) -> bool:
        item = self.get_item_by_id(item_id)
        if not item: return False

        children_ids = [child.id for child in self.items if child.parent_id == item_id]
        for child_id in children_ids:
            self.remove_item(child_id, record_history=record_history)

        self.items = [i for i in self.items if i.id != item_id]
        if self.current_item and self.current_item.id == item_id:
            self.current_item = None

        if record_history:
            self._add_history_entry(item, status_override=ActionStatus.CANCELLED, notes="Item removed")
        self._save_data()
        return True

    def set_current_item(self, item_id: str) -> ActionItem | None:
        item = self.get_item_by_id(item_id)
        if not item: return None
        if item.status == ActionStatus.COMPLETED and item.item_type == ItemType.TASK and item.frequency == Frequency.ONE_TIME:
            return None

        self.current_item = item
        if item.status == ActionStatus.NOT_STARTED:
            item.status = ActionStatus.IN_PROGRESS
            item.updated_at = datetime.now(pytz.utc)
            self._add_history_entry(item, notes="Set as current, status to In Progress")
        else:
            self._add_history_entry(item, notes="Set as current")
        self._save_data()
        return item

    def complete_current_item(self) -> ActionItem | None:
        if not self.current_item: return None

        item_to_complete = self.current_item
        item_to_complete.status = ActionStatus.COMPLETED
        item_to_complete.last_completed = datetime.now(pytz.utc)
        item_to_complete.updated_at = datetime.now(pytz.utc)

        self._recalculate_next_due(item_to_complete)
        self._add_history_entry(item_to_complete, status_override=ActionStatus.COMPLETED, notes="Marked as completed")

        self.current_item = None  # Clear current item after completion
        self._save_data()
        return item_to_complete

    def get_suggestions(self, count: int = 2) -> list[ActionItem]:
        # Prioritize AI suggestions if ISAA is available
        if self.isaa:
            active_items_for_ai = []
            for item in self.items:
                if item.status != ActionStatus.COMPLETED and item.status != ActionStatus.CANCELLED:
                    # Convert datetimes to user's local timezone string for AI context
                    item_dump = item.model_dump_json_safe()  # This is already UTC ISO
                    # Optionally, convert to user's timezone string if AI is better with local times
                    # For now, UTC ISO is fine.
                    active_items_for_ai.append(item_dump)

            MAX_ITEMS_FOR_CONTEXT = 20
            if len(active_items_for_ai) > MAX_ITEMS_FOR_CONTEXT:
                active_items_for_ai.sort(
                    key=lambda x: (x.get('priority', 3), x.get('next_due') or '9999-12-31T23:59:59Z'))
                active_items_for_ai = active_items_for_ai[:MAX_ITEMS_FOR_CONTEXT]

            now_user_tz_str = datetime.now(self.get_user_timezone()).isoformat()

            prompt = (
                f"User's current time: {now_user_tz_str} (Timezone: {self.settings.timezone}). "
                f"Active items (tasks/notes) are provided below (datetimes are in UTC ISO format). "
                f"Suggest the top {count} item IDs to focus on. Consider priority, due dates (next_due), "
                f"and if a current item is set (current_item_id), its sub-items might be relevant. "
                f"Tasks are generally more actionable. Focus on 'not_started' or 'in_progress'.\n\n"
                f"Active Items (JSON):\n{json.dumps(active_items_for_ai, indent=2)}\n\n"
                f"Current Item ID: {self.current_item.id if self.current_item else 'None'}\n\n"
                f"Return JSON: {{ \"suggested_item_ids\": [\"id1\", \"id2\"] }}."
            )

            class SuggestedIds(BaseModel):
                suggested_item_ids: list[str]

            try:
                structured_response = asyncio.run(
                    self.isaa.format_class(SuggestedIds, prompt, agent_name="TaskCompletion"))
                if structured_response and isinstance(structured_response, dict):
                    suggested_ids_model = SuggestedIds(**structured_response)
                    ai_suggestions = [self.get_item_by_id(id_str) for id_str in suggested_ids_model.suggested_item_ids
                                      if self.get_item_by_id(id_str)]
                    if ai_suggestions: return ai_suggestions[:count]
            except Exception as e:
                self.app.logger.error(f"Error getting AI suggestions: {e}")

        # Fallback to basic suggestions
        return self._get_basic_suggestions(count)

    def _get_basic_suggestions(self, count: int = 2) -> list[ActionItem]:
        now_utc = datetime.now(pytz.utc)
        available_items = [
            item for item in self.items
            if item.status in [ActionStatus.NOT_STARTED, ActionStatus.IN_PROGRESS]
        ]

        if self.current_item:
            sub_items = [item for item in available_items if item.parent_id == self.current_item.id]
            # If current item has actionable sub-items, prioritize them
            if any(s.next_due and s.next_due < (now_utc + timedelta(hours=2)) for s in sub_items) or \
                any(s.priority <= 2 for s in sub_items):  # Urgent sub-items (due soon or high priority)
                available_items = sub_items  # Focus on sub-items
            # If no urgent sub-items, consider other items too, but maybe give slight preference to other sub-items.
            # For simplicity now, if current_item is set, and it has sub-items, suggestions come from sub-items.
            # If no sub-items, or current_item is not set, consider all available_items.
            elif sub_items:  # Has sub-items, but none are "urgent" by above criteria
                available_items = sub_items
            # If current_item has no sub_items, then general pool is used.

        def sort_key(item: ActionItem):
            # Sort by: 1. Due Date (earlier is better, None is last) 2. Priority (lower num is higher)
            due_date_utc = item.next_due if item.next_due else datetime.max.replace(tzinfo=pytz.utc)
            return (due_date_utc, item.priority)

        available_items.sort(key=sort_key)
        return available_items[:count]

    def get_history(self, limit: int = 50) -> list[HistoryEntry]:
        return sorted(self.history, key=lambda x: x.timestamp, reverse=True)[:limit]

    def get_all_items_hierarchy(self) -> dict[str, list[dict[str, Any]]]:
        # This method remains largely the same, just ensure model_dump_json_safe is used.
        # Datetimes will be ISO UTC strings. Client JS needs to handle display in user's local time.
        hierarchy = {"root": []}
        item_map = {item.id: item.model_dump_json_safe() for item in self.items}  # Uses UTC ISO dates

        # This part seems fine, it builds hierarchy based on parent_id
        processed_ids = set()
        root_items_temp = []

        for _item_id, item_dict in item_map.items():
            parent_id = item_dict.get("parent_id")
            if parent_id and parent_id in item_map:
                if "children" not in item_map[parent_id]:
                    item_map[parent_id]["children"] = []
                item_map[parent_id]["children"].append(item_dict)
            else:
                root_items_temp.append(item_dict)
        hierarchy["root"] = root_items_temp

        def sort_children_recursive(node_list):
            for node_dict in node_list:
                if "children" in node_dict:
                    # Sort children by priority, then creation date
                    node_dict["children"].sort(key=lambda x: (x.get('priority', 3), isoparse(x.get('created_at'))))
                    sort_children_recursive(node_dict["children"])

        # Sort root items
        hierarchy["root"].sort(key=lambda x: (x.get('priority', 3), isoparse(x.get('created_at'))))
        sort_children_recursive(hierarchy["root"])
        return hierarchy

    # --- AI Specific Methods ---
    async def ai_create_item_from_text(self, text: str) -> ActionItem | None:
        if not self.isaa:
            self.app.logger.warning("ISAA module not available for AI item creation.")
            return None

        class ParsedItemFromText(BaseModel):
            item_type: Literal["task", "note"] = "task"
            title: str
            description: str | None = None
            priority: int | None = Field(default=3, ge=1, le=5)
            due_date_str: str | None = None  # e.g., "tomorrow", "next monday at 5pm", "2024-12-25 17:00"
            frequency_str: str | None = Field(default="one_time",
                                                 description="e.g. 'daily', 'weekly', 'one_time', 'every friday'")

        user_tz = self.get_user_timezone()
        current_time_user_tz_str = datetime.now(user_tz).strftime('%Y-%m-%d %H:%M:%S %Z%z')
        prompt = (
            f"User's current time is {current_time_user_tz_str}. Parse the input into a structured item. "
            f"For due_date_str, interpret relative dates/times based on this current time and output "
            f"a specific date string like 'YYYY-MM-DD HH:MM:SS'. If time is omitted, assume a default like 9 AM. "
            f"If date is omitted but time is given (e.g. 'at 5pm'), assume today if 5pm is future, else tomorrow. "
            f"User input: \"{text}\"\n\n"
            f"Format as JSON for ParsedItemFromText."
        )
        try:
            raw_response = await self.isaa.mini_task_completion(prompt, agent_name="TaskCompletion")
            if not raw_response: self.app.logger.error("AI parsing returned empty."); return None

            json_str = raw_response
            if "```json" in json_str: json_str = json_str.split("```json")[1].split("```")[0].strip()
            parsed_dict = json.loads(json_str)
            parsed_data_model = ParsedItemFromText(**parsed_dict)

            item_constructor_data = {
                "item_type": ItemType(parsed_data_model.item_type),
                "title": parsed_data_model.title,
                "description": parsed_data_model.description,
                "priority": parsed_data_model.priority or 3,
            }

            if parsed_data_model.due_date_str:
                # ISAA is prompted to return YYYY-MM-DD HH:MM:SS.
                # This string is assumed to be in the user's local timezone.
                # The ActionItem model_validator will convert this to UTC.
                item_constructor_data["fixed_time"] = parsed_data_model.due_date_str  # Pass as string

            # Frequency parsing (simplified)
            if parsed_data_model.frequency_str:
                freq_str_lower = parsed_data_model.frequency_str.lower()
                if "daily" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.DAILY
                elif "weekly" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.WEEKLY
                elif "monthly" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.MONTHLY
                elif "annually" in freq_str_lower or "yearly" in freq_str_lower:
                    item_constructor_data["frequency"] = Frequency.ANNUALLY
                else:
                    item_constructor_data["frequency"] = Frequency.ONE_TIME

            return self.add_item(item_constructor_data, by_ai=True)
        except Exception as e:
            self.app.logger.error(
                f"Error creating item with AI: {e}. Raw: {raw_response if 'raw_response' in locals() else 'N/A'}")
            return None

    def _log_ai_action(self, action_type: Literal["ai_create_item", "ai_modify_item", "ical_import"],
                       item_ids: list[str], previous_data_map: dict[str, str] | None = None):
        entry = UndoLogEntry(action_type=action_type, item_ids=item_ids, previous_data_json_map=previous_data_map)
        self.undo_log.append(entry)
        if len(self.undo_log) > 20: self.undo_log = self.undo_log[-20:]
        # _save_data called by caller

    async def undo_last_ai_action(self) -> bool:  # Also handles iCal import undo
        if not self.undo_log: return False
        last_action = self.undo_log.pop()
        action_undone_count = 0

        if last_action.action_type in ["ai_create_item", "ical_import"]:
            for item_id in last_action.item_ids:
                if self.remove_item(item_id, record_history=False):  # Don't double-log removal for undo
                    action_undone_count += 1
        elif last_action.action_type == "ai_modify_item":
            if last_action.previous_data_json_map:
                for item_id, prev_data_json in last_action.previous_data_json_map.items():
                    try:
                        prev_data = ActionItem.model_validate_json_safe(json.loads(prev_data_json),
                                                                        user_timezone_str=self.settings.timezone)
                        # Replace item
                        found = False
                        for i, item_in_list in enumerate(self.items):
                            if item_in_list.id == item_id:
                                self.items[i] = prev_data
                                if self.current_item and self.current_item.id == item_id:
                                    self.current_item = prev_data
                                found = True
                                break
                        if found:
                            action_undone_count += 1
                        else:
                            self.app.logger.warning(f"Could not find item {item_id} to restore during AI undo.")
                    except Exception as e:
                        self.app.logger.error(f"Error restoring item {item_id} during undo: {e}")
            else:  # Should not happen for modify
                self.app.logger.warning(
                    f"Undo for AI modify action on item(s) {last_action.item_ids} had no previous_data_json_map.")

        if action_undone_count > 0:
            # Create a generic history entry for the undo action
            generic_undo_item_title = f"Related to {len(last_action.item_ids)} item(s)"
            if len(last_action.item_ids) == 1:
                item_for_title = self.get_item_by_id(last_action.item_ids[0])  # Might be None if it was a create undo
                generic_undo_item_title = item_for_title.title if item_for_title else "N/A (Undone Action)"

            self.history.append(HistoryEntry(
                item_id=last_action.item_ids[0],  # Representative item
                item_title=generic_undo_item_title,
                item_type=ItemType.TASK,  # Generic
                status_changed_to=ActionStatus.CANCELLED,  # Generic status for undo
                notes=f"Undid action: {last_action.action_type} for {len(last_action.item_ids)} item(s)."
            ))
            self._save_data()
            return True

        # If nothing was undone, put action back to log
        self.undo_log.append(last_action)
        return False

    # --- iCalendar Methods ---
    def _parse_ical_dt(self, dt_ical: vDatetime | vDate, user_tz: pytz.BaseTzInfo) -> datetime | None:
        """Converts icalendar vDatetime or vDate to UTC datetime."""
        if not dt_ical: return None
        dt_val = dt_ical.dt

        if isinstance(dt_val, datetime):
            if dt_val.tzinfo is None:  # Naive datetime, assume user's local timezone as per iCal spec for floating
                return user_tz.localize(dt_val).astimezone(pytz.utc)
            return dt_val.astimezone(pytz.utc)  # Aware datetime
        elif isinstance(dt_val, date):  # All-day event, represent as start of day in user's TZ, then UTC
            return user_tz.localize(datetime.combine(dt_val, datetime.min.time())).astimezone(pytz.utc)
        return None

    def _map_ical_priority_to_app(self, ical_priority: int | None) -> int:
        if ical_priority is None: return 3  # Default
        if 1 <= ical_priority <= 4: return 1  # High
        if ical_priority == 5: return 3  # Medium
        if 6 <= ical_priority <= 9: return 5  # Low
        return 3  # Default for 0 or other values

    def _map_app_priority_to_ical(self, app_priority: int) -> int:
        if app_priority == 1: return 1  # High
        if app_priority == 2: return 3
        if app_priority == 3: return 5  # Medium
        if app_priority == 4: return 7
        if app_priority == 5: return 9  # Low
        return 0  # No priority

    def _map_rrule_to_frequency(self, rrule_prop: vRecur | None) -> tuple[Frequency, str | None]:
        if not rrule_prop:
            return Frequency.ONE_TIME, None

        rrule_dict = rrule_prop.to_dict()
        freq = rrule_dict.get('FREQ')
        original_rrule_str = vRecur.from_dict(rrule_dict).to_ical().decode('utf-8')

        if freq == 'DAILY': return Frequency.DAILY, original_rrule_str
        if freq == 'WEEKLY': return Frequency.WEEKLY, original_rrule_str
        if freq == 'MONTHLY': return Frequency.MONTHLY, original_rrule_str
        if freq == 'YEARLY': return Frequency.ANNUALLY, original_rrule_str

        # If RRULE is complex or not a direct match, import as ONE_TIME for each instance
        # but store the original RRULE string for reference or future advanced handling.
        return Frequency.ONE_TIME, original_rrule_str

    def import_ical_events(self, ical_string: str) -> list[ActionItem]:
        imported_items: list[ActionItem] = []
        try:
            cal = iCalCalendar.from_ical(ical_string)
            user_tz = self.get_user_timezone()
            now_utc = datetime.now(pytz.utc)
            import_limit_date_utc = now_utc + timedelta(days=RECURRING_IMPORT_WINDOW_DAYS)

            processed_uids_for_session = set()  # To avoid processing same base recurring event multiple times in one import

            for component in cal.walk():
                if component.name == "VEVENT":
                    uid = component.get('uid')
                    if not uid:
                        uid = str(uuid.uuid4())  # Generate a UID if missing
                    else:
                        uid = uid.to_ical().decode('utf-8')

                    summary = component.get('summary', 'Untitled Event').to_ical().decode('utf-8')
                    description = component.get('description', '').to_ical().decode('utf-8')
                    location = component.get('location', '').to_ical().decode('utf-8')
                    dtstart_ical = component.get('dtstart')
                    dtend_ical = component.get('dtend')  # Can be used for duration if needed
                    ical_priority_val = component.get('priority')
                    ical_priority = int(ical_priority_val.to_ical().decode('utf-8')) if ical_priority_val else None

                    rrule_prop = component.get('rrule')  # This is a vRecur object or None

                    start_time_utc = self._parse_ical_dt(dtstart_ical, user_tz)
                    if not start_time_utc:
                        self.app.logger.warning(f"Skipping event '{summary}' due to missing/invalid DTSTART.")
                        continue

                    app_priority = self._map_ical_priority_to_app(ical_priority)

                    # Check for existing item with this iCal UID to potentially update (simplistic check)
                    # A more robust update would involve comparing sequence numbers, etc.
                    # For now, if UID exists, we might skip or update. Let's try to update.
                    # To keep it simpler for now, we will create new items for occurrences.
                    # UID management needs to be precise for updates.
                    # If an item is an instance of a recurring event, its UID in our system might be base_uid + occurrence_date.

                    if rrule_prop:
                        if uid in processed_uids_for_session:  # Already processed this recurring event's base
                            continue
                        processed_uids_for_session.add(uid)

                        # Handle recurring event
                        rrule_str = rrule_prop.to_ical().decode('utf-8')
                        # Ensure DTSTART is part of the rrule context if not explicitly in rrulestr
                        if 'DTSTART' not in rrule_str.upper() and start_time_utc:
                            # dateutil.rrule needs start time; icalendar often bakes it in.
                            # If start_time_utc is naive, use user_tz to make it aware.
                            dtstart_for_rrule = start_time_utc.astimezone(
                                user_tz) if start_time_utc.tzinfo else user_tz.localize(start_time_utc)
                            # rrule_obj = rrulestr(rrule_str, dtstart=dtstart_for_rrule) # This is complex due to TZ handling in rrulestr
                            # The icalendar library's component should be timezone aware from DTSTART
                            # So, let's assume dtstart_ical.dt is the correct starting point.
                            try:
                                rrule_obj = rrulestr(rrule_str, dtstart=dtstart_ical.dt)
                            except Exception as e_rr:
                                self.app.logger.error(
                                    f"Could not parse RRULE '{rrule_str}' for event '{summary}': {e_rr}")
                                continue

                        occurrences_imported = 0
                        # Generate occurrences starting from now (in user's timezone, aligned to event's time)
                        # or from event's start_time_utc if it's in the future.

                        # The rrule iteration should be in the event's original timezone context if possible,
                        # or consistently in user's timezone for 'now'.
                        # Let's use UTC for iteration and then convert.

                        # Iterate from the event's actual start time or now, whichever is later for relevant future instances.
                        iteration_start_utc = max(now_utc, start_time_utc)

                        for occ_dt_aware in rrule_obj.between(iteration_start_utc, import_limit_date_utc, inc=True):
                            if occurrences_imported >= MAX_RECURRING_INSTANCES_TO_IMPORT:
                                break

                            # occ_dt_aware is usually from dateutil.rrule, may need tzinfo set or conversion.
                            # If rrulestr was given an aware dtstart, occurrences should be aware.
                            # Ensure it's UTC for our system.
                            occ_utc = occ_dt_aware.astimezone(pytz.utc) if occ_dt_aware.tzinfo else pytz.utc.localize(
                                occ_dt_aware)

                            instance_uid = f"{uid}-{occ_utc.strftime('%Y%m%dT%H%M%S%Z')}"

                            # Check if this specific instance already exists
                            existing_instance = next((item for item in self.items if item.ical_uid == instance_uid),
                                                     None)
                            if existing_instance:
                                self.app.logger.info(
                                    f"Instance {instance_uid} for '{summary}' already exists. Skipping.")
                                continue

                            item_data = {
                                "title": summary, "description": description, "location": location,
                                "item_type": ItemType.TASK, "fixed_time": occ_utc,
                                "frequency": Frequency.ONE_TIME,  # Each imported instance is one-time in our system
                                "priority": app_priority, "ical_uid": instance_uid,  # Instance-specific UID
                                "status": ActionStatus.NOT_STARTED,
                                "ical_rrule_original": rrule_str  # Store original rule for reference
                            }
                            new_item = self.add_item(item_data, imported=True)
                            imported_items.append(new_item)
                            occurrences_imported += 1

                        if occurrences_imported == 0 and start_time_utc > now_utc and start_time_utc <= import_limit_date_utc:
                            # If it's a future non-recurring event (or rrule didn't yield instances in window but start is in window)
                            # This case is for when rrule_prop exists but yields no instances in the .between() range,
                            # but the initial DTSTART itself is valid and upcoming.
                            # However, rrule.between should include dtstart if inc=True and it's within range.
                            # This path might be redundant if .between is inclusive and dtstart is in range.
                            pass


                    else:  # Non-recurring event
                        # Only import if it's upcoming or started recently and not completed (e.g. within last day)
                        if start_time_utc < (
                            now_utc - timedelta(days=1)) and not dtend_ical:  # Too old, and no end time to check
                            self.app.logger.info(f"Skipping old non-recurring event '{summary}' (UID: {uid})")
                            continue
                        if dtend_ical:
                            end_time_utc = self._parse_ical_dt(dtend_ical, user_tz)
                            if end_time_utc and end_time_utc < now_utc:  # Event has already ended
                                self.app.logger.info(f"Skipping past event '{summary}' (UID: {uid}) that has ended.")
                                continue

                        existing_item = next((item for item in self.items if item.ical_uid == uid), None)
                        if existing_item:  # Simplistic update: remove old, add new. Better: update in place.
                            self.app.logger.info(
                                f"Event with UID {uid} ('{summary}') already exists. Re-importing (simple replace).")
                            self.remove_item(existing_item.id, record_history=False)

                        item_data = {
                            "title": summary, "description": description, "location": location,
                            "item_type": ItemType.TASK, "fixed_time": start_time_utc,
                            "frequency": Frequency.ONE_TIME, "priority": app_priority,
                            "ical_uid": uid, "status": ActionStatus.NOT_STARTED
                        }
                        new_item = self.add_item(item_data, imported=True)
                        imported_items.append(new_item)

            if imported_items:
                self._log_ai_action("ical_import", [item.id for item in imported_items])
            self._save_data()  # Ensure all changes are saved
            self.app.logger.info(f"Imported {len(imported_items)} items from iCalendar data.")

        except Exception as e:
            self.app.logger.error(f"Failed to parse iCalendar string: {e}", exc_info=True)
            # Potentially re-raise or return empty list with error status
        return imported_items

    def import_ical_from_url(self, url: str) -> list[ActionItem]:
        try:
            headers = {'User-Agent': 'POA_App/1.0 (+https://yourdomain.com/poa_app_info)'}  # Be a good internet citizen
            response = requests.get(url, timeout=10, headers=headers)
            response.raise_for_status()  # Raises HTTPError for bad responses (4XX or 5XX)
            return self.import_ical_events(response.text)
        except requests.exceptions.RequestException as e:
            self.app.logger.error(f"Error fetching iCalendar from URL {url}: {e}")
            return []
        except Exception as e:  # Catch other errors like parsing
            self.app.logger.error(f"Error processing iCalendar from URL {url}: {e}")
            return []

    def import_ical_from_file_content(self, file_content: bytes) -> list[ActionItem]:
        try:
            # Try to decode as UTF-8, but iCal can have other encodings.
            # Standard is UTF-8. `icalendar` lib handles encoding detection mostly.
            ical_string = file_content.decode('utf-8', errors='replace')
            return self.import_ical_events(ical_string)
        except UnicodeDecodeError as e:
            self.app.logger.error(f"Encoding error reading iCalendar file: {e}. Try ensuring UTF-8 encoding.")
            # Try with 'latin-1' as a common fallback for some older files
            try:
                ical_string = file_content.decode('latin-1', errors='replace')
                return self.import_ical_events(ical_string)
            except Exception as e_fallback:
                self.app.logger.error(f"Fallback decoding also failed for iCalendar file: {e_fallback}")
                return []
        except Exception as e:
            self.app.logger.error(f"Error processing iCalendar file content: {e}")
            return []

    def export_to_ical_string(self) -> str:
        cal = iCalCalendar()
        cal.add('prodid', '-//POA App//yourdomain.com//')
        cal.add('version', '2.0')
        user_tz = self.get_user_timezone()

        for item in self.items:
            if item.item_type == ItemType.TASK and item.fixed_time:
                event = iCalEvent()
                event.add('summary', item.title)

                # Ensure fixed_time is UTC for iCal standard practice
                dtstart_utc = item.fixed_time
                if dtstart_utc.tzinfo is None:  # Should not happen if stored correctly
                    dtstart_utc = pytz.utc.localize(dtstart_utc)
                else:
                    dtstart_utc = dtstart_utc.astimezone(pytz.utc)
                event.add('dtstart', dtstart_utc)  # vDatetime handles UTC conversion for .to_ical()

                # Add DTEND (e.g., 1 hour duration for tasks, or based on item if available)
                # For simplicity, let's assume 1 hour duration if not specified
                event.add('dtend', dtstart_utc + timedelta(hours=1))

                event.add('dtstamp', datetime.now(pytz.utc))  # Time the event was created in iCal
                event.add('uid', item.ical_uid or item.id)  # Use original iCal UID if present, else our ID

                if item.description:
                    event.add('description', item.description)
                if item.location:
                    event.add('location', item.location)

                event.add('priority', self._map_app_priority_to_ical(item.priority))

                # Handle recurrence
                if item.frequency != Frequency.ONE_TIME:
                    if item.ical_rrule_original:  # If we have the original complex rule, use it
                        try:
                            # vRecur.from_ical requires bytes
                            event.add('rrule', vRecur.from_ical(item.ical_rrule_original.encode()))
                        except Exception as e_rrule:
                            self.app.logger.warning(
                                f"Could not parse stored original RRULE '{item.ical_rrule_original}' for item {item.id}: {e_rrule}. Exporting as simple recurrence.")
                            # Fallback to simple mapping
                            self._add_simple_rrule(event, item.frequency)
                    else:  # Map simple frequency
                        self._add_simple_rrule(event, item.frequency)

                cal.add_component(event)
        return cal.to_ical().decode('utf-8')

    def _add_simple_rrule(self, event: iCalEvent, frequency: Frequency):
        rrule_params = {}
        if frequency == Frequency.DAILY:
            rrule_params['freq'] = 'DAILY'
        elif frequency == Frequency.WEEKLY:
            rrule_params['freq'] = 'WEEKLY'
        elif frequency == Frequency.MONTHLY:
            rrule_params['freq'] = 'MONTHLY'
        elif frequency == Frequency.ANNUALLY:
            rrule_params['freq'] = 'YEARLY'

        if rrule_params:
            event.add('rrule', vRecur(rrule_params))

SchedulerManager

SchedulerManagerClass

Source code in toolboxv2/mods/SchedulerManager.py
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
class SchedulerManagerClass:
    def __init__(self):
        self.jobs = {}
        self.thread = None
        self.running = False
        self.last_successful_jobs = deque(maxlen=3)  # Stores last 3 successful job names
        self.job_errors = {}  # Stores job names as keys and error messages as values

    def _run(self):
        while self.running:
            schedule.run_pending()
            time.sleep(1)

    def start(self):
        if not self.running:
            self.running = True
            self.thread = threading.Thread(target=self._run, daemon=True)
            self.thread.start()

    def stop(self):
        self.running = False
        if self.thread is not None:
            self.thread.join()

    def job_wrapper(self, job_name: str, job_function: callable):
        """
        Wrap a job function to track success and errors.
        """
        def wrapped_job(*args, **kwargs):
            try:
                job_function(*args, **kwargs)
                # If the job ran successfully, store it in the success queue
                self.last_successful_jobs.append(job_name)
                if job_name in self.job_errors:
                    del self.job_errors[job_name]  # Remove error record if job succeeded after failing
            except Exception as e:
                # Capture any exceptions and store them
                self.job_errors[job_name] = str(e)

        return wrapped_job


    def register_job(self,
                     job_id: str,
                     second: int = -1,
                     func: (Callable or str) | None = None,
                     job: schedule.Job | None = None,
                     time_passer: schedule.Job | None = None,
                     object_name: str | None = None,
                     receive_job: bool = False,
                     save: bool = False,
                     max_live: bool = False,
                     serializer=serializer_default,
                     args=None, kwargs=None):
        """
            Parameters
            ----------
                job_id : str
                    id for the job for management
                second : int
                    The time interval in seconds between each call of the job.
                func : Callable or str
                    The function to be executed as the job.
                job : schedule.Job
                    An existing job object from the schedule library.
                time_passer : schedule.Job
                    A job without a function, used to specify the time interval.
                object_name : str
                    The name of the object containing in the 'func' var to be executed.
                receive_job : bool
                    A flag indicating whether the job should be received from an object from 'func' var.
                save : bool
                    A flag indicating whether the job should be saved.
                max_live : bool
                    A flag indicating whether the job should have a maximum live time.
                serializer : dill
                    json pickel or dill must have a dumps fuction
                *args, **kwargs : Any serializable and deserializable
                    Additional arguments to be passed to the job function.

            Returns
            -------
           """

        if job is None and func is None:
            return Result.default_internal_error("Both job and func are not specified."
                                                 " Please specify either job or func.")
        if job is not None and func is not None:
            return Result.default_internal_error("Both job and func are specified. Please specify either job or func.")

        if job is not None:
            def func(x):
                return x
            return self._save_job(job_id=job_id,
                                  job=job,
                                  save=save,
                                  func=func,
                                  args=args,
                                  kwargs=kwargs,
                                  serializer=serializer)

        parsed_attr = self._parse_function(func=func, object_name=object_name)

        if parsed_attr.is_error():
            parsed_attr.result.data_info = f"Error parsing function for job : {job_id}"
            return parsed_attr

        if receive_job:
            job = parsed_attr.get()
        else:
            func = parsed_attr.get()

        time_passer = self._prepare_time_passer(time_passer=time_passer,
                                                second=second)

        job_func = self._prepare_job_func(func=func,
                                          max_live=max_live,
                                          second=second,
                                          args=args,
                                          kwargs=kwargs,
                                          job_id=job_id)

        job = self._get_final_job(job=job,
                                  func=self.job_wrapper(job_id, job_func),
                                  time_passer=time_passer,
                                  job_func=job_func,
                                  args=args,
                                  kwargs=kwargs)
        if job.is_error():
            return job

        job = job.get()

        return self._save_job(job_id=job_id,
                              job=job,
                              save=save,
                              func=func,
                              args=args,
                              kwargs=kwargs,
                              serializer=serializer)

    @staticmethod
    def _parse_function(func: str or Callable, object_name):
        if isinstance(func, str) and func.endswith('.py'):
            with open(func) as file:
                func_code = file.read()
                exec(func_code)
                func = locals()[object_name]
        elif isinstance(func, str) and func.endswith('.dill') and safety_mode == 'open':
            try:
                with open(func, 'rb') as file:
                    func = dill.load(file)
            except FileNotFoundError:
                return Result.default_internal_error(f"Function file {func} not found or dill not installed")
        elif isinstance(func, str):
            local_vars = {'app': get_app(from_=Name + f".pasing.{object_name}")}
            try:
                exec(func.strip(), {}, local_vars)
            except Exception as e:
                return Result.default_internal_error(f"Function parsing failed withe {e}")
            func = local_vars[object_name]
        elif isinstance(func, Callable):
            pass
        else:
            return Result.default_internal_error("Could not parse object scheduler_manager.parse_function")
        return Result.ok(func)

    @staticmethod
    def _prepare_time_passer(time_passer, second):
        if time_passer is None and second > 0:
            return schedule.every(second).seconds
        elif time_passer is None and second <= 0:
            raise ValueError("second must be greater than 0")
        return time_passer

    def _prepare_job_func(self, func: Callable, max_live: bool, second: float, job_id: str, *args, **kwargs):
        if max_live:
            end_time = datetime.now() + timedelta(seconds=second)

            def job_func():
                if datetime.now() < end_time:
                    func(*args, **kwargs)
                else:
                    job = self.jobs.get(job_id, {}).get('job')
                    if job is not None:
                        schedule.cancel_job(job)
                    else:
                        print("Error Canceling job")

            return job_func
        return func

    @staticmethod
    def _get_final_job(job, func, time_passer, job_func, args, kwargs):
        if job is None and isinstance(func, Callable):
            job = time_passer.do(job_func, *args, **kwargs)
        elif job is not None:
            pass
        else:
            return Result.default_internal_error("No Final job found for register")
        return Result.ok(job)

    def _save_job(self, job_id, job, save, args=None, **kwargs):
        if job is not None:
            self.jobs[job_id] = {'id': job_id, 'job': job, 'save': save, 'func': job_id, 'args': args,
                                 'kwargs': kwargs}
            f = (f"Added Job {job_id} :{' - saved' if save else ''}"
                  f"{' - args ' + str(len(args)) if args else ''}"
                  f"{' - kwargs ' + str(len(kwargs.keys())) if kwargs else ''}")
            return Result.ok(f)
        else:
            return Result.default_internal_error(job_id)

    def cancel_job(self, job_id):
        if job_id not in self.jobs:
            print("Job not found")
            return
        schedule.cancel_job(self.jobs[job_id].get('job'))
        self.jobs[job_id]["cancelled"] = True
        self.jobs[job_id]["save"] = False
        print("Job cancelled")

    def del_job(self, job_id):
        if job_id not in self.jobs:
            print("Job not found")
            return
        if not self.jobs[job_id].get("cancelled", False):
            print("Job not cancelled canceling job")
            self.cancel_job(job_id)
        del self.jobs[job_id]
        print("Job deleted")

    def save_jobs(self, file_path, serializer=serializer_default):
        with open(file_path, 'wb') as file:
            save_jobs = [job for job in self.jobs.values() if job['save']]
            serializer.dump(save_jobs, file)

    def load_jobs(self, file_path, deserializer=deserializer_default):
        with open(file_path, 'rb') as file:
            jobs = deserializer.load(file)
            for job_info in jobs:
                del job_info['job']
                func = deserializer.loads(job_info['func'])
                self.register_job(job_info['id'], func=func, **job_info)

    def get_tasks_table(self):
        if not self.jobs:
            return "No tasks registered."

        # Calculate the maximum width for each column
        id_width = max(len("Task ID"), max(len(job_id) for job_id in self.jobs))
        next_run_width = len("Next Execution")
        interval_width = len("Interval")

        # Create the header
        header = f"| {'Task ID':<{id_width}} | {'Next Execution':<{next_run_width}} | {'Interval':<{interval_width}} |"
        separator = f"|{'-' * (id_width + 2)}|{'-' * (next_run_width + 2)}|{'-' * (interval_width + 2)}|"

        # Create the table rows
        rows = []
        for job_id, job_info in self.jobs.items():
            job = job_info['job']
            next_run = job.next_run.strftime("%Y-%m-%d %H:%M:%S") if job.next_run else "N/A"
            interval = self._get_interval_str(job)
            row = f"| {job_id:<{id_width}} | {next_run:<{next_run_width}} | {interval:<{interval_width}} |"
            rows.append(row)

        # Combine all parts of the table
        table = "\n".join([header, separator] + rows)
        return table

    def _get_interval_str(self, job):
        if job.interval == 0:
            return "Once"

        units = [
            (86400, "day"),
            (3600, "hour"),
            (60, "minute"),
            (1, "second")
        ]

        for seconds, unit in units:
            if job.interval % seconds == 0:
                count = job.interval // seconds
                return f"Every {count} {unit}{'s' if count > 1 else ''}"

        return f"Every {job.interval} seconds"
job_wrapper(job_name, job_function)

Wrap a job function to track success and errors.

Source code in toolboxv2/mods/SchedulerManager.py
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def job_wrapper(self, job_name: str, job_function: callable):
    """
    Wrap a job function to track success and errors.
    """
    def wrapped_job(*args, **kwargs):
        try:
            job_function(*args, **kwargs)
            # If the job ran successfully, store it in the success queue
            self.last_successful_jobs.append(job_name)
            if job_name in self.job_errors:
                del self.job_errors[job_name]  # Remove error record if job succeeded after failing
        except Exception as e:
            # Capture any exceptions and store them
            self.job_errors[job_name] = str(e)

    return wrapped_job
register_job(job_id, second=-1, func=None, job=None, time_passer=None, object_name=None, receive_job=False, save=False, max_live=False, serializer=serializer_default, args=None, kwargs=None)
Parameters
job_id : str
    id for the job for management
second : int
    The time interval in seconds between each call of the job.
func : Callable or str
    The function to be executed as the job.
job : schedule.Job
    An existing job object from the schedule library.
time_passer : schedule.Job
    A job without a function, used to specify the time interval.
object_name : str
    The name of the object containing in the 'func' var to be executed.
receive_job : bool
    A flag indicating whether the job should be received from an object from 'func' var.
save : bool
    A flag indicating whether the job should be saved.
max_live : bool
    A flag indicating whether the job should have a maximum live time.
serializer : dill
    json pickel or dill must have a dumps fuction
*args, **kwargs : Any serializable and deserializable
    Additional arguments to be passed to the job function.
Returns
Source code in toolboxv2/mods/SchedulerManager.py
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
def register_job(self,
                 job_id: str,
                 second: int = -1,
                 func: (Callable or str) | None = None,
                 job: schedule.Job | None = None,
                 time_passer: schedule.Job | None = None,
                 object_name: str | None = None,
                 receive_job: bool = False,
                 save: bool = False,
                 max_live: bool = False,
                 serializer=serializer_default,
                 args=None, kwargs=None):
    """
        Parameters
        ----------
            job_id : str
                id for the job for management
            second : int
                The time interval in seconds between each call of the job.
            func : Callable or str
                The function to be executed as the job.
            job : schedule.Job
                An existing job object from the schedule library.
            time_passer : schedule.Job
                A job without a function, used to specify the time interval.
            object_name : str
                The name of the object containing in the 'func' var to be executed.
            receive_job : bool
                A flag indicating whether the job should be received from an object from 'func' var.
            save : bool
                A flag indicating whether the job should be saved.
            max_live : bool
                A flag indicating whether the job should have a maximum live time.
            serializer : dill
                json pickel or dill must have a dumps fuction
            *args, **kwargs : Any serializable and deserializable
                Additional arguments to be passed to the job function.

        Returns
        -------
       """

    if job is None and func is None:
        return Result.default_internal_error("Both job and func are not specified."
                                             " Please specify either job or func.")
    if job is not None and func is not None:
        return Result.default_internal_error("Both job and func are specified. Please specify either job or func.")

    if job is not None:
        def func(x):
            return x
        return self._save_job(job_id=job_id,
                              job=job,
                              save=save,
                              func=func,
                              args=args,
                              kwargs=kwargs,
                              serializer=serializer)

    parsed_attr = self._parse_function(func=func, object_name=object_name)

    if parsed_attr.is_error():
        parsed_attr.result.data_info = f"Error parsing function for job : {job_id}"
        return parsed_attr

    if receive_job:
        job = parsed_attr.get()
    else:
        func = parsed_attr.get()

    time_passer = self._prepare_time_passer(time_passer=time_passer,
                                            second=second)

    job_func = self._prepare_job_func(func=func,
                                      max_live=max_live,
                                      second=second,
                                      args=args,
                                      kwargs=kwargs,
                                      job_id=job_id)

    job = self._get_final_job(job=job,
                              func=self.job_wrapper(job_id, job_func),
                              time_passer=time_passer,
                              job_func=job_func,
                              args=args,
                              kwargs=kwargs)
    if job.is_error():
        return job

    job = job.get()

    return self._save_job(job_id=job_id,
                          job=job,
                          save=save,
                          func=func,
                          args=args,
                          kwargs=kwargs,
                          serializer=serializer)

Tools

Bases: MainTool, SchedulerManagerClass

Source code in toolboxv2/mods/SchedulerManager.py
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
class Tools(MainTool, SchedulerManagerClass):
    version = version

    def __init__(self, app=None):
        self.name = Name
        self.color = "VIOLET2"

        self.keys = {"mode": "db~mode~~:"}
        self.encoding = 'utf-8'
        self.tools = {'name': Name}

        SchedulerManagerClass.__init__(self)
        MainTool.__init__(self,
                          load=self.init_sm,
                          v=self.version,
                          name=self.name,
                          color=self.color,
                          on_exit=self.on_exit)


    @export(
        mod_name=Name,
        name="Version",
        version=version,
    )
    def get_version(self):
        return self.version

    # Exportieren der Scheduler-Instanz für die Nutzung in anderen Modulen
    @export(mod_name=Name, name='init', version=version, initial=True)
    def init_sm(self):
        if os.path.exists(self.app.data_dir + '/jobs.compact'):
            print("SchedulerManager try loading from file")
            self.load_jobs(
                self.app.data_dir + '/jobs.compact'
            )
            print("SchedulerManager Successfully loaded")
        print("STARTING SchedulerManager")
        self.start()

    @export(mod_name=Name, name='clos_manager', version=version, exit_f=True)
    def on_exit(self):
        self.stop()
        self.save_jobs(self.app.data_dir + '/jobs.compact')
        return f"saved {len(self.jobs.keys())} jobs in {self.app.data_dir + '/jobs.compact'}"

    @export(mod_name=Name, name='instance', version=version)
    def get_instance(self):
        return self

    @export(mod_name=Name, name='start', version=version)
    def start_instance(self):
        return self.start()

    @export(mod_name=Name, name='stop', version=version)
    def stop_instance(self):
        return self.stop()

    @export(mod_name=Name, name='cancel', version=version)
    def cancel_instance(self, job_id):
        return self.cancel_job(job_id)

    @export(mod_name=Name, name='dealt', version=version)
    def dealt_instance(self, job_id):
        return self.del_job(job_id)

    @export(mod_name=Name, name='add', version=version)
    def register_instance(self, job_data: dict):
        """
        example dicts :
            -----------
            {
                "job_id": "job0",
                "second": 0,
                "func": None,
                "job": None,
                "time_passer": None,
                "object_name": "tb_job_fuction",
                "receive_job": False,
                "save": False,
                "max_live": True,
                # just lev it out "serializer": serializer_default,
                "args": [],
                "kwargs": {},
            }

            job_id : str
                id for the job for management
            second (optional): int
                The time interval in seconds between each call of the job.
            func (optional): Callable or str
                The function to be executed as the job.
            job (optional):  schedule.Job
                An existing job object from the schedule library.
            time_passer (optional):  schedule.Job
                A job without a function, used to specify the time interval.
            object_name (optional): str
                The name of the object containing in the 'func' var to be executed.
            receive_job (optional): bool
                A flag indicating whether the job should be received from an object from 'func' var.
            save (optional): bool
                A flag indicating whether the job should be saved.
            max_live (optional): bool
                A flag indicating whether the job should have a maximum live time.
            serializer (optional): bool
                json pickel or dill must have a dumps fuction
            *args, **kwargs (optional):
                Additional arguments to be passed to the job function.


        Parameters
            ----------
           job_data : dict

        example usage
            ----------
            `python

            `

    """
        if job_data is None:
            self.app.logger.error("No job data provided")
            return None
        job_id = job_data["job_id"]
        second = job_data.get("second", 0)
        func = job_data.get("func")
        job = job_data.get("job")
        time_passer = job_data.get("time_passer")
        object_name = job_data.get("object_name", "tb_job_fuction")
        receive_job = job_data.get("receive_job", False)
        save = job_data.get("save", False)
        max_live = job_data.get("max_live", True)
        serializer = job_data.get("serializer", serializer_default)
        args = job_data.get("args", ())
        kwargs = job_data.get("kwargs", {})

        return self.register_job(
            job_id=job_id,
            second=second,
            func=func,
            job=job,
            time_passer=time_passer,
            object_name=object_name,
            receive_job=receive_job,
            save=save,
            max_live=max_live,
            serializer=serializer,
            args=args,
            kwargs=kwargs
        )
register_instance(job_data)
example dicts

{ "job_id": "job0", "second": 0, "func": None, "job": None, "time_passer": None, "object_name": "tb_job_fuction", "receive_job": False, "save": False, "max_live": True, # just lev it out "serializer": serializer_default, "args": [], "kwargs": {}, }

job_id : str id for the job for management second (optional): int The time interval in seconds between each call of the job. func (optional): Callable or str The function to be executed as the job. job (optional): schedule.Job An existing job object from the schedule library. time_passer (optional): schedule.Job A job without a function, used to specify the time interval. object_name (optional): str The name of the object containing in the 'func' var to be executed. receive_job (optional): bool A flag indicating whether the job should be received from an object from 'func' var. save (optional): bool A flag indicating whether the job should be saved. max_live (optional): bool A flag indicating whether the job should have a maximum live time. serializer (optional): bool json pickel or dill must have a dumps fuction args, *kwargs (optional): Additional arguments to be passed to the job function.

Parameters ---------- job_data : dict

example usage ---------- `python

`
Source code in toolboxv2/mods/SchedulerManager.py
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
@export(mod_name=Name, name='add', version=version)
def register_instance(self, job_data: dict):
    """
    example dicts :
        -----------
        {
            "job_id": "job0",
            "second": 0,
            "func": None,
            "job": None,
            "time_passer": None,
            "object_name": "tb_job_fuction",
            "receive_job": False,
            "save": False,
            "max_live": True,
            # just lev it out "serializer": serializer_default,
            "args": [],
            "kwargs": {},
        }

        job_id : str
            id for the job for management
        second (optional): int
            The time interval in seconds between each call of the job.
        func (optional): Callable or str
            The function to be executed as the job.
        job (optional):  schedule.Job
            An existing job object from the schedule library.
        time_passer (optional):  schedule.Job
            A job without a function, used to specify the time interval.
        object_name (optional): str
            The name of the object containing in the 'func' var to be executed.
        receive_job (optional): bool
            A flag indicating whether the job should be received from an object from 'func' var.
        save (optional): bool
            A flag indicating whether the job should be saved.
        max_live (optional): bool
            A flag indicating whether the job should have a maximum live time.
        serializer (optional): bool
            json pickel or dill must have a dumps fuction
        *args, **kwargs (optional):
            Additional arguments to be passed to the job function.


    Parameters
        ----------
       job_data : dict

    example usage
        ----------
        `python

        `

"""
    if job_data is None:
        self.app.logger.error("No job data provided")
        return None
    job_id = job_data["job_id"]
    second = job_data.get("second", 0)
    func = job_data.get("func")
    job = job_data.get("job")
    time_passer = job_data.get("time_passer")
    object_name = job_data.get("object_name", "tb_job_fuction")
    receive_job = job_data.get("receive_job", False)
    save = job_data.get("save", False)
    max_live = job_data.get("max_live", True)
    serializer = job_data.get("serializer", serializer_default)
    args = job_data.get("args", ())
    kwargs = job_data.get("kwargs", {})

    return self.register_job(
        job_id=job_id,
        second=second,
        func=func,
        job=job,
        time_passer=time_passer,
        object_name=object_name,
        receive_job=receive_job,
        save=save,
        max_live=max_live,
        serializer=serializer,
        args=args,
        kwargs=kwargs
    )

SocketManager

The SocketManager Supports 2 types of connections 1. Client Server 2. Peer to Peer

TruthSeeker

arXivCrawler

ArXiv Crawler for TruthSeeker. Main module for processing research queries.

ArXivPDFProcessor

Main processor for research queries. This is a wrapper around the new ResearchProcessor for backward compatibility.

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
class ArXivPDFProcessor:
    """
    Main processor for research queries.
    This is a wrapper around the new ResearchProcessor for backward compatibility.
    """
    def __init__(self,
                 query: str,
                 tools,
                 chunk_size: int = 1_000_000,
                 overlap: int = 2_000,
                 max_workers=None,
                 num_search_result_per_query=6,
                 max_search=6,
                 download_dir="pdfs",
                 callback=None,
                 num_workers=None):
        """Initialize the ArXiv PDF processor.

        Args:
            query: Research query
            tools: Tools module
            chunk_size: Size of text chunks for processing
            overlap: Overlap between chunks
            max_workers: Maximum number of worker threads
            num_search_result_per_query: Number of search results per query
            max_search: Maximum number of search queries
            download_dir: Directory to save downloaded files
            callback: Callback function for status updates
            num_workers: Number of worker threads
        """
        # Create the new research processor
        self.processor = ResearchProcessor(
            query=query,
            tools=tools,
            chunk_size=chunk_size,
            overlap=overlap,
            max_workers=max_workers,
            num_search_result_per_query=num_search_result_per_query,
            max_search=max_search,
            download_dir=download_dir,
            callback=callback,
            num_workers=num_workers
        )

        # Copy attributes for backward compatibility
        self.insights_generated = False
        self.queries_generated = False
        self.query = query
        self.tools = tools
        self.mem = tools.get_memory()
        self.chunk_size = chunk_size
        self.overlap = overlap
        self.max_workers = max_workers
        self.nsrpq = num_search_result_per_query
        self.max_search = max_search
        self.download_dir = download_dir
        self.parser = RobustPDFDownloader(download_dir=download_dir)
        self.callback = callback if callback is not None else lambda status: None
        self.mem_name = None
        self.current_session = None
        self.all_ref_papers = 0
        self.last_insights_list = None
        self.all_texts_len = 0
        self.f_texts_len = 0
        self.s_id = str(uuid.uuid4())
        self.semantic_model = self.processor.semantic_model
        self._query_progress = {}
        self._progress_lock = threading.Lock()
        self.num_workers = self.processor.num_workers

    def _update_global_progress(self) -> float:
        """Calculate overall progress considering all processing phases."""
        return self.processor._update_global_progress()

    async def search_and_process_papers(self, queries: list[str]) -> list[Paper]:
        """Search for and process papers based on queries.

        Args:
            queries: List of search queries

        Returns:
            List of processed papers
        """
        # Use the new processor to search and process papers
        unified_papers = await self.processor.search_and_process_papers(queries)

        # Convert UnifiedPaper objects to Paper objects for backward compatibility
        papers = []
        for paper in unified_papers:
            if paper.source == "arxiv":
                # Convert to the old Paper format
                arxiv_paper = Paper(
                    title=paper.title,
                    authors=paper.authors,
                    summary=paper.summary,
                    url=paper.url,
                    pdf_url=paper.pdf_url,
                    published=paper.published,
                    updated=paper.source_specific_data.get("updated", ""),
                    categories=paper.source_specific_data.get("categories", []),
                    paper_id=paper.paper_id
                )
                papers.append(arxiv_paper)

        # Update attributes for backward compatibility
        self.all_ref_papers = self.processor.all_ref_papers
        self.all_texts_len = self.processor.all_texts_len
        self.f_texts_len = self.processor.f_texts_len

        return papers

    def send_status(self, step: str, progress: float = None, additional_info: str = ""):
        """Send status update via callback."""
        if progress is None:
            progress = self._update_global_progress()
        self.callback({
            "step": step,
            "progress": progress,
            "info": additional_info
        })

    def generate_queries(self) -> list[str]:
        self.send_status("Generating search queries")
        self.queries_generated = False

        class ArXivQueries(BaseModel):
            queries: list[str] = Field(..., description="List of ArXiv search queries (en)")

        try:
            query_generator: ArXivQueries = self.tools.format_class(
                ArXivQueries,
                f"Generate a list of precise ArXiv search queries to comprehensively address: {self.query}"
            )
            queries = [self.query] + query_generator["queries"]
        except Exception:
            self.send_status("Error generating queries", additional_info="Using default query.")
            queries = [self.query]

        if len(queries[:self.max_search]) > 0:
            self.queries_generated = True
        return queries[:self.max_search]

    def init_process_papers(self):
        self.mem.create_memory(self.mem_name, model_config={"model_name": "anthropic/claude-3-5-haiku-20241022"})
        self.send_status("Memory initialized")


    async def generate_insights(self, queries) -> dict:
        self.send_status("Generating insights")
        query = self.query
        # max_it = 0
        results = await self.mem.query(query=query, memory_names=self.mem_name, unified_retrieve=True, query_params={
            "max_sentences": 25})
        #query = queries[min(len(queries)-1, max_it)]

        self.insights_generated = True
        self.send_status("Insights generated", progress=1.0)
        return results

    async def extra_query(self, query, query_params=None, unified_retrieve=True):
        self.send_status("Processing follow-up query", progress=0.5)
        results = await self.mem.query(query=query, memory_names=self.mem_name,
                                                      query_params=query_params, unified_retrieve=unified_retrieve)
        self.send_status("Processing follow-up query Done", progress=1)
        return results

    def generate_mem_name(self):
        class UniqueMemoryName(BaseModel):
            """unique memory name based on the user query"""
            name: str
        return self.tools.get_agent("thinkm").format_class(UniqueMemoryName, self.query).get('name', '_'.join(self.query.split(" ")[:3]))

    def initialize(self, session_id, second=False):
        self.current_session = session_id
        self.insights_generated = False
        self.queries_generated = False
        if second:
            return
        self.mem_name = self.generate_mem_name().strip().replace("\n", '') + '_' + session_id
        self.init_process_papers()

    async def process(self, query=None) -> tuple[list[Paper], dict]:
        if query is not None:
            self.query = query
        self.send_status("Starting research process")
        t0 = time.perf_counter()
        self.initialize(self.s_id, query is not None)

        queries = self.generate_queries()

        papers = await self.search_and_process_papers(queries)

        if len(papers) == 0:
            class UserQuery(BaseModel):
                """Fix all typos and clear the original user query"""
                new_query: str
            self.query= self.tools.format_class(
                UserQuery,
                self.query
            )["new_query"]
            queries = self.generate_queries()
            papers = await self.search_and_process_papers(queries)

        insights = await self.generate_insights(queries)

        elapsed_time = time.perf_counter() - t0
        self.send_status("Process complete", progress=1.0,
                         additional_info=f"Total time: {elapsed_time:.2f}s, Papers analyzed: {len(papers)}/{self.all_ref_papers}")

        return papers, insights

    @staticmethod
    def estimate_processing_metrics(query_length: int, **config) -> (float, float):
        """Return estimated time (seconds) and price for processing."""
        total_papers = config['max_search'] * config['num_search_result_per_query']
        median_text_length = 100000  # 10 pages * 10000 characters

        # Estimated chunks to process
        total_chunks = total_papers * (median_text_length / config['chunk_size']) + 1 / config['overlap']
        processed_chunks = total_chunks * 0.45
        total_chars = TextSplitter(config['chunk_size'],
                     config['overlap']
                     ).approximate(config['chunk_size'] * processed_chunks)
        # Time estimation (seconds)
        .75 / config['chunk_size']  # Hypothetical time per chunk in seconds
        w = (config.get('num_workers', 16) if config.get('num_workers', 16) is not None else 16 / 10)
        # Processing_ time - Insights Genration - Insights Query   -   Indexing Time     -    Download Time     -       workers   -   Query Genration time - Ui - Init Db
        estimated_time = ((8+total_papers*0.012)+(total_chunks/20000) * .005 + (total_chunks/2) * .0003 + total_papers * 2.8 ) / w + (0.25 * config['max_search']) + 6 + 4

        price_per_char = 0.0000012525
        price_per_t_chunk =  total_chars * price_per_char
        estimated_price = price_per_t_chunk ** 1.7

        # estimated_price = 0 if query_length < 420 and estimated_price < 5 else estimated_price
        if estimated_time < 10:
            estimated_time = 10
        if estimated_price < .04:
            estimated_price = .04
        return round(estimated_time, 2), round(estimated_price, 4)
__init__(query, tools, chunk_size=1000000, overlap=2000, max_workers=None, num_search_result_per_query=6, max_search=6, download_dir='pdfs', callback=None, num_workers=None)

Initialize the ArXiv PDF processor.

Parameters:

Name Type Description Default
query str

Research query

required
tools

Tools module

required
chunk_size int

Size of text chunks for processing

1000000
overlap int

Overlap between chunks

2000
max_workers

Maximum number of worker threads

None
num_search_result_per_query

Number of search results per query

6
max_search

Maximum number of search queries

6
download_dir

Directory to save downloaded files

'pdfs'
callback

Callback function for status updates

None
num_workers

Number of worker threads

None
Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
def __init__(self,
             query: str,
             tools,
             chunk_size: int = 1_000_000,
             overlap: int = 2_000,
             max_workers=None,
             num_search_result_per_query=6,
             max_search=6,
             download_dir="pdfs",
             callback=None,
             num_workers=None):
    """Initialize the ArXiv PDF processor.

    Args:
        query: Research query
        tools: Tools module
        chunk_size: Size of text chunks for processing
        overlap: Overlap between chunks
        max_workers: Maximum number of worker threads
        num_search_result_per_query: Number of search results per query
        max_search: Maximum number of search queries
        download_dir: Directory to save downloaded files
        callback: Callback function for status updates
        num_workers: Number of worker threads
    """
    # Create the new research processor
    self.processor = ResearchProcessor(
        query=query,
        tools=tools,
        chunk_size=chunk_size,
        overlap=overlap,
        max_workers=max_workers,
        num_search_result_per_query=num_search_result_per_query,
        max_search=max_search,
        download_dir=download_dir,
        callback=callback,
        num_workers=num_workers
    )

    # Copy attributes for backward compatibility
    self.insights_generated = False
    self.queries_generated = False
    self.query = query
    self.tools = tools
    self.mem = tools.get_memory()
    self.chunk_size = chunk_size
    self.overlap = overlap
    self.max_workers = max_workers
    self.nsrpq = num_search_result_per_query
    self.max_search = max_search
    self.download_dir = download_dir
    self.parser = RobustPDFDownloader(download_dir=download_dir)
    self.callback = callback if callback is not None else lambda status: None
    self.mem_name = None
    self.current_session = None
    self.all_ref_papers = 0
    self.last_insights_list = None
    self.all_texts_len = 0
    self.f_texts_len = 0
    self.s_id = str(uuid.uuid4())
    self.semantic_model = self.processor.semantic_model
    self._query_progress = {}
    self._progress_lock = threading.Lock()
    self.num_workers = self.processor.num_workers
estimate_processing_metrics(query_length, **config) staticmethod

Return estimated time (seconds) and price for processing.

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
@staticmethod
def estimate_processing_metrics(query_length: int, **config) -> (float, float):
    """Return estimated time (seconds) and price for processing."""
    total_papers = config['max_search'] * config['num_search_result_per_query']
    median_text_length = 100000  # 10 pages * 10000 characters

    # Estimated chunks to process
    total_chunks = total_papers * (median_text_length / config['chunk_size']) + 1 / config['overlap']
    processed_chunks = total_chunks * 0.45
    total_chars = TextSplitter(config['chunk_size'],
                 config['overlap']
                 ).approximate(config['chunk_size'] * processed_chunks)
    # Time estimation (seconds)
    .75 / config['chunk_size']  # Hypothetical time per chunk in seconds
    w = (config.get('num_workers', 16) if config.get('num_workers', 16) is not None else 16 / 10)
    # Processing_ time - Insights Genration - Insights Query   -   Indexing Time     -    Download Time     -       workers   -   Query Genration time - Ui - Init Db
    estimated_time = ((8+total_papers*0.012)+(total_chunks/20000) * .005 + (total_chunks/2) * .0003 + total_papers * 2.8 ) / w + (0.25 * config['max_search']) + 6 + 4

    price_per_char = 0.0000012525
    price_per_t_chunk =  total_chars * price_per_char
    estimated_price = price_per_t_chunk ** 1.7

    # estimated_price = 0 if query_length < 420 and estimated_price < 5 else estimated_price
    if estimated_time < 10:
        estimated_time = 10
    if estimated_price < .04:
        estimated_price = .04
    return round(estimated_time, 2), round(estimated_price, 4)
search_and_process_papers(queries) async

Search for and process papers based on queries.

Parameters:

Name Type Description Default
queries list[str]

List of search queries

required

Returns:

Type Description
list[Paper]

List of processed papers

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
async def search_and_process_papers(self, queries: list[str]) -> list[Paper]:
    """Search for and process papers based on queries.

    Args:
        queries: List of search queries

    Returns:
        List of processed papers
    """
    # Use the new processor to search and process papers
    unified_papers = await self.processor.search_and_process_papers(queries)

    # Convert UnifiedPaper objects to Paper objects for backward compatibility
    papers = []
    for paper in unified_papers:
        if paper.source == "arxiv":
            # Convert to the old Paper format
            arxiv_paper = Paper(
                title=paper.title,
                authors=paper.authors,
                summary=paper.summary,
                url=paper.url,
                pdf_url=paper.pdf_url,
                published=paper.published,
                updated=paper.source_specific_data.get("updated", ""),
                categories=paper.source_specific_data.get("categories", []),
                paper_id=paper.paper_id
            )
            papers.append(arxiv_paper)

    # Update attributes for backward compatibility
    self.all_ref_papers = self.processor.all_ref_papers
    self.all_texts_len = self.processor.all_texts_len
    self.f_texts_len = self.processor.f_texts_len

    return papers
send_status(step, progress=None, additional_info='')

Send status update via callback.

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py
322
323
324
325
326
327
328
329
330
def send_status(self, step: str, progress: float = None, additional_info: str = ""):
    """Send status update via callback."""
    if progress is None:
        progress = self._update_global_progress()
    self.callback({
        "step": step,
        "progress": progress,
        "info": additional_info
    })
main(query='Beste strategien in bretspielen sitler von katar') async

Main execution function

Source code in toolboxv2/mods/TruthSeeker/arXivCrawler.py
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
async def main(query: str = "Beste strategien in bretspielen sitler von katar"):
    """Main execution function"""
    with Spinner("Init Isaa"):
        tools = get_app("ArXivPDFProcessor", name=None).get_mod("isaa")
        tools.init_isaa(build=True)
    processor = ArXivPDFProcessor(query, tools=tools)
    papers, insights = await processor.process()

    print("Generated Insights:", insights)
    print("Generated Insights_list:", processor.last_insights_list)
    kb = tools.get_memory(processor.mem_name)
    print(await kb.query_concepts("AI"))
    print(await kb.retrieve("Evaluation metrics for assessing AI Agent performance"))
    print(kb.concept_extractor.concept_graph.concepts.keys())
    kb.vis(output_file="insights_graph.html")
    kb.save("mem.plk")
    # await get_app("ArXivPDFProcessor", name=None).a_idle()
    return insights

nGui

import colorsys import json import time from datetime import datetime, timedelta from queue import Queue from typing import Dict, Union, List, Any

import os import random from threading import Thread, Event

import networkx as nx from dataclasses import asdict

from toolboxv2 import get_app from toolboxv2.mods.FastApi.fast_nice import register_nicegui

import asyncio

from nicegui import ui

from pathlib import Path import stripe

from toolboxv2.mods.TruthSeeker.arXivCrawler import Paper from toolboxv2.mods.isaa.base.AgentUtils import anything_from_str_to_dict

Set your secret key (use environment variables in production!)

stripe.api_key = os.getenv('STRIPE_SECRET_KEY', 'sk_test_YourSecretKey')

def create_landing_page(): # Set up dynamic background ui.query("body").style("background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%)")

# Main container with enhanced responsive design
with ui.column().classes(
"w-full max-w-md p-8 rounded-3xl shadow-2xl "
"items-center self-center mx-auto my-8"
):
    # Advanced styling for glass-morphism effect
    ui.query(".nicegui-column").style("""
    background: rgba(255, 255, 255, 0.05);
    backdrop-filter: blur(12px);
    border: 1px solid rgba(255, 255, 255, 0.1);
    transition: all 0.3s ease-in-out;
    """)

    # Animated logo/brand icon
    with ui.element("div").classes("animate-fadeIn"):
        ui.icon("science").classes(
        "text-7xl mb-6 text-primary "
        "transform hover:scale-110 transition-transform"
        )

    # Enhanced typography for title
    ui.label("TruthSeeker").classes(
    "text-5xl font-black text-center "
    "text-primary mb-2 animate-slideDown"
    )

    # Stylized subtitle with brand message
    ui.label("Precision. Discovery. Insights.").classes(
    "text-xl font-medium text-center "
    "mb-10 animate-fadeIn"
    )

    # Button container for consistent spacing
    ui.button(
    "Start Research",
    on_click=lambda: ui.navigate.to("/open-Seeker.seek")
    ).classes(
    "w-full px-6 py-4 text-lg font-bold "
    "bg-primary hover:bg-primary-dark "
    "transform hover:-translate-y-0.5 "
    "transition-all duration-300 ease-in-out "
    "rounded-xl shadow-lg animate-slideUp"
    )

    # Navigation links container
    with ui.element("div").classes("mt-8 space-y-3 text-center"):
        ui.link(
        "Demo video",
        ).classes(
        "block text-lg text-gray-200 hover:text-primary "
        "transition-colors duration-300 animate-fadeIn"
        ).on("click", lambda: ui.navigate.to("/open-Seeker.demo"))

        ui.link(
        "About Us",
        ).classes(
        "block text-lg text-gray-400 hover:text-primary "
        "transition-colors duration-300 animate-fadeIn"
        ).on("click", lambda: ui.navigate.to("/open-Seeker.about"))

def create_video_demo(): with ui.card().classes('w-full max-w-3xl mx-auto').style( 'background: var(--background-color); color: var(--text-color)'): # Video container with responsive aspect ratio with ui.element('div').classes('relative w-full aspect-video'): video = ui.video('../api/TruthSeeker/video').classes('w-full h-full object-cover')

        # Custom controls overlay
        with ui.element('div').classes('absolute bottom-0 left-0 right-0 bg-black/50 p-2'):
            with ui.row().classes('items-center gap-2'):
                #play_btn = ui.button(icon='play_arrow', on_click=lambda: video.props('playing=true'))
                #pause_btn = ui.button(icon='pause', on_click=lambda: video.props('playing=false'))
                ui.slider(min=0, max=100, value=0).classes('w-full').bind_value(video, 'time')
                #mute_btn = ui.button(icon='volume_up', on_click=lambda: video.props('muted=!muted'))
                #fullscreen_btn = ui.button(icon='fullscreen', on_click=lambda: video.props('fullscreen=true'))


    # Video description
    ui.markdown('Walkthrough of TruthSeeker features and capabilities.')
    # Back to Home Button
    ui.button('Back to Home', on_click=lambda: ui.navigate.to('/open-Seeker')).classes(
        'mt-6 w-full bg-primary text-white hover:opacity-90'
    )

return video

def create_about_page(): """Create a comprehensive About page for TruthSeeker""" with ui.column().classes('w-full max-w-4xl mx-auto p-6'): # Page Header ui.label('About TruthSeeker').classes('text-4xl font-bold text-primary mb-6')

    # Mission Statement
    with ui.card().classes('w-full mb-6').style(
        'background: var(--background-color); color: var(--text-color); padding: 20px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);'
    ):
        ui.label('Our Mission').classes('text-2xl font-semibold text-primary mb-4')
        ui.markdown("""
            TruthSeeker aims to democratize access to scientific knowledge,
            transforming complex academic research into comprehensible insights.
            We bridge the gap between raw data and meaningful understanding.
        """).classes('text-lg').style('color: var(--text-color);')

    # Core Technologies
    with ui.card().classes('w-full mb-6').style(
        'background: var(--background-color); color: var(--text-color); padding: 20px; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);'
    ):
        ui.label('Core Technologies').classes('text-2xl font-semibold text-primary mb-4')
        with ui.row().classes('gap-4 w-full'):
            with ui.column().classes('flex-1 text-center'):
                ui.icon('search').classes('text-4xl text-primary mb-2')
                ui.label('Advanced Query Processing').classes('font-bold')
                ui.markdown('Intelligent algorithms that extract nuanced research insights.').style(
                    'color: var(--text-color);')
            with ui.column().classes('flex-1 text-center'):
                ui.icon('analytics').classes('text-4xl text-primary mb-2')
                ui.label('Semantic Analysis').classes('font-bold')
                ui.markdown('Deep learning models for comprehensive research verification.').style(
                    'color: var(--text-color);')
            with ui.column().classes('flex-1 text-center'):
                ui.icon('verified').classes('text-4xl text-primary mb-2')
                ui.label('Research Validation').classes('font-bold')
                ui.markdown('Multi-layered verification of academic sources.').style('color: var(--text-color);')
    # Research Process
    with ui.card().classes('w-full').style('background: var(--background-color);color: var(--text-color);'):
        ui.label('Research Discovery Process').classes('text-2xl font-semibold text-primary mb-4')
        with ui.card().classes('q-pa-md q-mx-auto').style(
            'max-width: 800px; background: var(--background-color); border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.1);'
        ) as card:
            ui.markdown("# Research Workflow").style(
                "color: var(--primary-color); text-align: center; margin-bottom: 20px;")
            ui.markdown(
                """
                Welcome to TruthSeeker’s interactive research assistant. Follow the steps below to transform your initial inquiry into a refined, actionable insight.
                """
            ).style("color: var(--text-color); text-align: center; margin-bottom: 30px;")

            # The stepper component
            with ui.stepper().style('background: var(--background-color); color: var(--text-color);') as stepper:
                # Step 1: Query Initialization
                with ui.step('Query Initialization'):
                    ui.markdown("### Step 1: Query Initialization").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        Begin by entering your research question or selecting from popular academic domains.
                        This sets the direction for our semantic analysis engine.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Next', on_click=stepper.next).props('rounded color=primary')

                # Step 2: Semantic Search
                with ui.step('Semantic Search'):
                    ui.markdown("### Step 2: Semantic Search").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        Our advanced algorithms now process your input to generate context-rich queries.
                        This stage refines the search context by understanding the deeper intent behind your question.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Back', on_click=stepper.previous).props('flat')
                        ui.button('Next', on_click=stepper.next).props('rounded color=primary')

                # Step 3: Document Analysis
                with ui.step('Document Analysis'):
                    ui.markdown("### Step 3: Document Analysis").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        The system then dives into a detailed analysis of academic papers, parsing content to extract key insights and connections.
                        This ensures that even subtle but crucial information is captured.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Back', on_click=stepper.previous).props('flat')
                        ui.button('Next', on_click=stepper.next).props('rounded color=primary')

                # Step 4: Insight Generation
                with ui.step('Insight Generation'):
                    ui.markdown("### Step 4: Insight Generation").style("color: var(--primary-color);")
                    ui.markdown(
                        """
                        Finally, we synthesize the analyzed data into clear, actionable research summaries.
                        These insights empower you with concise guidance to drive further inquiry or practical application.
                        """
                    ).style("color: var(--text-color); margin-bottom: 20px;")
                    with ui.stepper_navigation():
                        ui.button('Back', on_click=stepper.previous).props('flat')

    # Back to Home Button
    ui.button('Back to Home', on_click=lambda: ui.navigate.to('/open-Seeker')).classes(
        'mt-6 w-full bg-primary text-white hover:opacity-90'
    )
Dummy-Implementierung für get_tools()

def get_tools(): """ Hier solltest du dein richtiges Werkzeug-Objekt zurückliefern. In diesem Beispiel gehen wir davon aus, dass du über eine Funktion wie get_app verfügst. """ return get_app("ArXivPDFProcessor", name=None).get_mod("isaa")

def create_graph_tab(processor_instance: Dict, graph_ui: ui.element, main_ui: ui.element): """Create and update the graph visualization"""

# Get HTML graph from processor
_html_content = processor_instance["instance"].tools.get_memory(processor_instance["instance"].mem_name)
html_content = "" if isinstance(_html_content, list) else _html_content.vis(get_output_html=True)

# Ensure static directory exists
static_dir = Path('dist/static')
static_dir.mkdir(exist_ok=True)

# Save HTML to static file
graph_file = static_dir / f'graph{processor_instance["instance"].mem_name}.html'
# Save HTML to static file with added fullscreen functionality

# Add fullscreen JavaScript
graph_file.write_text(html_content, encoding='utf-8')

with main_ui:
    # Clear existing content except fullscreen button
    graph_ui.clear()

    with graph_ui:
        ui.html(f"""

            <iframe
                 src="/static/graph{processor_instance["instance"].mem_name}.html"
                style="width: 100%; height: 800px; border: none; background: #1a1a1a;"
                >
            </iframe>
        """).classes('w-full h-full')

is_init = [False]

--- Database Setup ---

def get_db(): db = get_app().get_mod("DB") if not is_init[0]: is_init[0] = True db.edit_cli("LD") db.initialize_database() return db

import pickle

--- Session State Management ---

def get_user_state(session_id: str, is_new=False) -> dict: db = get_db() state_ = { 'balance': .5, 'last_reset': datetime.utcnow().isoformat(), 'research_history': [], 'payment_id': '', } if session_id is None: state_['balance'] *= -1 if is_new: return state_, True return state_ state = db.get(f"TruthSeeker::session:{session_id}") if state.get() is None: state = state_ if is_new: return state_, True else: try: state = pickle.loads(state.get()) except Exception as e: print(e) state = { 'balance': 0.04, 'last_reset': datetime.utcnow().isoformat(), 'research_history': ["Sorry we had an error recreating your state"], 'payment_id': '', } if is_new: return state, True if is_new: return state, False return state

def save_user_state(session_id: str, state: dict): db = get_db() print("Saving state") db.set(f"TruthSeeker::session:{session_id}", pickle.dumps(state)).print()

def delete_user_state(session_id: str): db = get_db() print("Saving state") db.delete(f"TruthSeeker::session:{session_id}").print()

def reset_daily_balance(state: dict, valid=False) -> dict: now = datetime.utcnow() last_reset = datetime.fromisoformat(state.get('last_reset', now.isoformat())) if now - last_reset > timedelta(hours=24): state['balance'] = max(state.get('balance', 1.6 if valid else 0.5), 1.6 if valid else 0.5) state['last_reset'] = now.isoformat() return state

class MemoryResultsDisplay

def init(self, results: List[Dict[str, Any]], main_ui: ui.element): self.results = results self.main_ui = main_ui self.setup_ui()

def setup_ui(self): """Set up the main UI for displaying memory results""" with self.main_ui: self.main_ui.clear() with ui.column().classes('w-full'): for mem_result in self.results: self.create_memory_card(mem_result)

def create_memory_card(self, mem_result: Dict[str, Any]): """Create a card for each memory result""" result = mem_result.get("result", {}) with self.main_ui: if isinstance(result, dict): self.display_dict_result(result) elif hasattr(result, 'overview'): # Assuming RetrievalResult type self.display_retrieval_result(result) else: ui.label("Unsupported result type").classes('--text-color:error')

def display_dict_result(self, result: Dict[str, Any]): """Display dictionary-based result with collapsible sections""" # Summary Section summary = result.get("summary", {}) if isinstance(summary, str): try: summary = json.loads(summary[:-1]) except json.JSONDecodeError: summary = {"error": "Could not parse summary"}

# Raw Results Section
raw_results = result.get("raw_results", {})
if isinstance(raw_results, str):
    try:
        raw_results = json.loads(raw_results[:-1])
    except json.JSONDecodeError:
        raw_results = {"error": "Could not parse raw results"}

# Metadata Section
metadata = result.get("metadata", {})
with self.main_ui:
    # Collapsible Sections
    with ui.column().classes('w-full space-y-2').style("max-width: 100%;"):
        # Summary Section
        with ui.expansion('Summary', icon='description').classes('w-full') as se:
            self.display_nested_data(summary, main_ui=se)

        # Raw Results Section
        with ui.expansion('Raw Results', icon='work').classes('w-full') as re:
            self.display_nested_data(raw_results, main_ui=re)

        # Metadata Section
        if metadata:
            with ui.expansion('Metadata', icon='info').classes('w-full'):
                ui.markdown(f"```json

{json.dumps(metadata, indent=2)} ```").style("max-width: 100%;")

def display_retrieval_result(self, result):
    """Display retrieval result with detailed sections"""
    with self.main_ui:
        with ui.column().classes('w-full space-y-4').style("max-width: 100%;"):
            # Overview Section
            with ui.expansion('Overview', icon='visibility').classes('w-full') as ov:
                for overview_item in result.overview:
                    if isinstance(overview_item, str):
                        overview_item = json.loads(overview_item)
                    self.display_nested_data(overview_item, main_ui=ov)

            # Details Section
            with ui.expansion('Details', icon='article').classes('w-full'):
                for chunk in result.details:
                    with ui.card().classes('w-full p-3 mb-2').style("background: var(--background-color)"):
                        ui.label(chunk.text).classes('font-medium mb-2 --text-color:secondary')

                        with ui.row().classes('w-full justify-between').style("background: var(--background-color)"):
                            ui.label(f"Embedding Shape: {chunk.embedding.shape}").classes('text-sm')
                            ui.label(f"Content Hash: {chunk.content_hash}").classes('text-sm')

                        if chunk.cluster_id is not None:
                            ui.label(f"Cluster ID: {chunk.cluster_id}").classes('text-sm')

            # Cross References Section
            with ui.expansion('Cross References', icon='link').classes('w-full'):
                for topic, chunks in result.cross_references.items():
                    with ui.card().classes('w-full p-3 mb-2').style("background: var(--background-color)"):
                        ui.label(topic).classes('font-semibold mb-2 --text-color:secondary')
                        for chunk in chunks:
                            ui.label(chunk.text).classes('text-sm mb-1')

def display_nested_data(self, data: Union[Dict, List], indent: int = 0, main_ui=None):
    """Recursively display nested dictionary or list data"""
    with (self.main_ui if main_ui is None else main_ui):
        if isinstance(data, dict):
            with ui.column().classes(f'ml-{indent * 2}').style("max-width: 100%;"):
                for key, value in data.items():
                    with ui.row().classes('items-center'):
                        ui.label(f"{key}:").classes('font-bold mr-2 --text-color:primary')
                        if isinstance(value, list):
                            if key == "main_chunks":
                                continue
                            self.display_nested_data(value, indent + 1, main_ui=main_ui)
                        if isinstance(value, dict):
                            ui.markdown(f"```json

{json.dumps(value, indent=2)} ").classes("break-words w-full").style("max-width: 100%;") else: ui.label(str(value)).classes('--text-color:secondary') elif isinstance(data, list): with ui.column().classes(f'ml-{indent * 2}').style("max-width: 100%;"): for item in data: if isinstance(item, str): item = json.loads(item) if isinstance(item, list): self.display_nested_data(item, indent + 1, main_ui=main_ui) if isinstance(item, dict): ui.markdown(f"json {json.dumps(item, indent=2)} ```").classes("break-words w-full").style("max-width: 100%;") else: ui.label(str(item)).classes('--text-color:secondary')

def create_followup_section(processor_instance: Dict, main_ui: ui.element, session_id, balance): main_ui.clear() with main_ui: ui.label("Query Interface (1ct)").classes("text-xl font-semibold mb-4")

    # Container for query inputs
    query_container = ui.column().classes("w-full gap-4")
    query = ""  # Store references to query inputs
    # Query parameters section
    with ui.expansion("Query Parameters", icon="settings").classes("w-full") as query_e:
        with ui.grid(columns=2).classes("w-full gap-4"):
            k_input = ui.number("Results Count (k)", value=2, min=1, max=20)
            min_sim = ui.number("Min Similarity", value=.3, min=0, max=1, step=0.1)
            cross_depth = ui.number("Cross Reference Depth", value=2, min=1, max=5)
            max_cross = ui.number("Max Cross References", value=10, min=1, max=20)
            max_sent = ui.number("Max Sentences", value=10, min=1, max=50)
            unified = ui.switch("Unified Retrieve (+3ct)", value=True)

    # Results display
    with ui.element("div").classes("w-full mt-4") as results_display:
        pass
    results_display = results_display
    with query_container:
        query_input = ui.input("Query", placeholder="Enter your query...")                 .classes("w-full")
    # Control buttons
    with ui.row().classes("w-full gap-4 mt-4"):
        ui.button("Execute Query", on_click=lambda: asyncio.create_task(execute_query()))                 .classes("bg-green-600 hover:bg-green-700")
        ui.button("Clear Results", on_click=lambda: results_display.clear())                 .classes("bg-red-600 hover:bg-red-700")
query_input = query_input

async def execute_query():
    """Execute a single query with parameters"""
    nonlocal query_input, results_display, main_ui
    try:
        query_text = query_input.value
        if not query_text.strip():
            with main_ui:
                ui.notify("No Input", type="warning")
            return ""

        if not processor_instance.get("instance"):
            with main_ui:
                ui.notify("No active processor instance", type="warning")
            return
        # Collect parameters
        params = {
            "k": int(k_input.value),
            "min_similarity": min_sim.value,
            "cross_ref_depth": int(cross_depth.value),
            "max_cross_refs": int(max_cross.value),
            "max_sentences": int(max_sent.value),
            "unified": unified.value
        }
        # Construct query parameters
        query_params = {
            "k": params["k"],
            "min_similarity": params["min_similarity"],
            "cross_ref_depth": params["cross_ref_depth"],
            "max_cross_refs": params["max_cross_refs"],
            "max_sentences": params["max_sentences"]
        }

        # Execute query
        results = await processor_instance["instance"].extra_query(
            query=query_text,
            query_params=query_params,
            unified_retrieve=params["unified"]
        )
        print("results",results)
        s = get_user_state(session_id)
        s['balance'] -= .04 if unified.value else .01
        save_user_state(session_id, s)
        with main_ui:
            balance.set_text(f"Balance: {s['balance']:.2f}€")
        # Format results
        with main_ui:
            with results_display:
                MemoryResultsDisplay(results, results_display)

    except Exception as e:
        return f"Error executing query: {str(e)}

"

# Add initial query input

online_states = [0] def create_research_interface(Processor):

def helpr(request, session: dict):

    state = {'balance':0, 'research_history': []}
    main_ui = None
    with ui.column().classes("w-full max-w-6xl mx-auto p-6 space-y-6") as loading:
        ui.spinner(size='lg')
        ui.label('Initializing...').classes('ml-2')

    # Container for main content (initially hidden)
    content = ui.column().classes('hidden')

    # Extract session data before spawning thread
    session_id = session.get('ID')
    session_id_h = session.get('IDh')
    session_rid = request.row.query_params.get('session_id') if hasattr(request, 'row') else request.query_params.get('session_id')
    session_valid = session.get('valid')

    # Thread communication
    result_queue = Queue()
    ready_event = Event()

    def init_background():
        nonlocal session_id, session_id_h, session_rid, session_valid
        try:
            # Original initialization logic
            _state, is_new = get_user_state(session_id, is_new=True)

            if is_new and session_id_h != "#0":
                _state = get_user_state(session_id_h)
                save_user_state(session_id, _state)
                delete_user_state(session_id_h)
            if session_rid:
                state_: dict
                state_, is_new_ = get_user_state(session_rid, is_new=True)
                if not is_new_:
                    _state = state_.copy()
                    state_['payment_id'] = ''
                    state_['last_reset'] = datetime.utcnow().isoformat()
                    state_['research_history'] = state_['research_history'][:3]
                    state_['balance'] = 0
                    save_user_state(session_id, _state)
            _state = reset_daily_balance(_state, session_valid)
            save_user_state(session_id, _state)

            # Send result back to main thread
            result_queue.put(_state)
            ready_event.set()
        except Exception as e:
            result_queue.put(e)
            ready_event.set()

        # Start background initialization

    Thread(target=init_background).start()

    def check_ready():
        nonlocal state
        if ready_event.is_set():
            result = result_queue.get()

            # Check if initialization failed
            if isinstance(result, Exception):
                loading.clear()
                with loading:
                    ui.label(f"Error during initialization: {str(result)}").classes('text-red-500')
                return

            # Get state and build main UI
            state = result
            loading.classes('hidden')
            content.classes(remove='hidden')
            main_ui.visible = True
            with main_ui:
                balance.set_text(f"Balance: {state['balance']:.2f}€")
                show_history()
            return  # Stop the timer

        # Check again in 100ms
        ui.timer(0.1, check_ready, once=True)

    # Start checking for completion
    check_ready()

    # Wir speichern die aktive Instanz, damit Follow-Up Fragen gestellt werden können
    processor_instance = {"instance": None}

    # UI-Elemente als Platzhalter; wir definieren sie später in der UI und machen sie so
    # in den Callback-Funktionen über "nonlocal" verfügbar.
    overall_progress = None
    status_label = None
    results_card = None
    summary_content = None
    analysis_content = None
    references_content = None
    followup_card = None
    research_card = None
    config_cart = None
    progress_card = None
    balance = None
    graph_ui = None

    sr_button = None
    r_button = None
    r_text = None


    # Global config storage with default values
    config = {
        'chunk_size': 21000,
        'overlap': 600,
        'num_search_result_per_query': 3,
        'max_search': 3,
        'num_workers': None
    }

    def update_estimates():
        """
        Dummy estimation based on query length and configuration.
        (Replace with your own non-linear formula if needed.)
        """
        query_text = query.value or ""
        query_length = len(query_text)
        # For example: estimated time scales with chunk size and query length.
        estimated_time ,estimated_price = Processor.estimate_processing_metrics(query_length, **config)
        estimated_time *= max(1, online_states[0] * 6)
        if processor_instance["instance"] is not None:
            estimated_price += .25
        if estimated_time < 60:
            time_str = f"~{int(estimated_time)}s"
        elif estimated_time < 3600:
            minutes = estimated_time // 60
            seconds = estimated_time % 60
            time_str = f"~{int(minutes)}m {int(seconds)}s"
        else:
            hours = estimated_time // 3600
            minutes = (estimated_time % 3600) // 60
            time_str = f"~{int(hours)}h {int(minutes)}m"
        with main_ui:
            query_length_label.set_text(f"Total Papers: {config['max_search']*config['num_search_result_per_query']}")
            time_label.set_text(f"Processing Time: {time_str}")
            price_label.set_text(f"Price: {estimated_price:.2f}€")

        return estimated_price

    def on_config_change(event):
        """
        Update the global config based on input changes and recalc estimates.
        """
        try:
            config['chunk_size'] = int(chunk_size_input.value)
        except ValueError:
            pass
        try:
            config['overlap'] = int(overlap_input.value)
            if config['overlap'] > config['chunk_size'] / 4:
                config['overlap'] = int(config['chunk_size'] / 4)
                with main_ui:
                    overlap_input.value = config['overlap']
        except ValueError:
            pass
        try:
            config['num_search_result_per_query'] = int(num_search_result_input.value)
        except ValueError:
            pass
        try:
            config['max_search'] = int(max_search_input.value)
        except ValueError:
            pass
        try:
            config['num_workers'] = int(num_workers_input.value) if num_workers_input.value != 0 else None
        except ValueError:
            config['num_workers'] = None

        update_estimates()

    def on_query_change():
        update_estimates()

    # Callback, der vom Processor (über processor_instance.callback) aufgerufen wird.
    def update_status(data: dict):
        nonlocal overall_progress, status_label
        if not data:
            return
        # Aktualisiere den Fortschrittsbalken und den aktuellen Schritt (wenn vorhanden)
        with main_ui:
            if isinstance(data, dict):
                progress = data.get("progress", 0)
                step = data.get("step", "Processing...")
                overall_progress.value =round( progress ,2) # nicegui.linear_progress erwartet einen Wert zwischen 0 und 1
                status_label.set_text(f"{step} {data.get('info','')}")
            else:
                status_label.set_text(f"{data}")

    def start_search():
        nonlocal balance

        async def helper():
            nonlocal processor_instance, overall_progress, status_label, results_card,                     summary_content, analysis_content,config, references_content, followup_card,sr_button,r_button,r_text

            try:
                if not validate_inputs():
                    with main_ui:
                        state['balance'] += est_price
                        save_user_state(session_id, state)
                        balance.set_text(f"Balance: {state['balance']:.2f}€")
                    return
                reset_interface()
                show_progress_indicators()

                query_text = query.value.strip()
                # Erzeuge das "tools"-Objekt (abhängig von deiner konkreten Implementation)
                tools = get_tools()
                with main_ui:
                    research_card.visible = False
                    config_cart.visible = False
                    config_section.visible = False
                    query.set_value("")
                # Direkt instanziieren: Eine neue ArXivPDFProcessor-Instanz
                if processor_instance["instance"] is not None:
                    processor = processor_instance["instance"]
                    processor.chunk_size = config['chunk_size']
                    processor.overlap = config['overlap']
                    processor.num_search_result_per_query = config['num_search_result_per_query']
                    processor.max_search = config['max_search']
                    processor.num_workers = config['num_workers']
                    papers, insights = await processor.process(query_text)
                else:
                    processor = Processor(query_text, tools=tools, **config)
                # Setze den Callback so, dass Updates in der GUI angezeigt werden
                    processor.callback = update_status
                    processor_instance["instance"] = processor
                    papers, insights = await processor.process()

                update_results({
                    "papers": papers,
                    "insights": insights
                })
                with main_ui:
                    research_card.visible = True
                    config_cart.visible = True
                    show_history()

            except Exception as e:
                import traceback

                with main_ui:
                    update_status({"progress": 0, "step": "Error", "info": str(e)})
                    state['balance'] += est_price
                    save_user_state(session_id, state)
                    balance.set_text(f"Balance: {state['balance']:.2f}€")
                    ui.notify(f"Error {str(e)})", type="negative")
                    research_card.visible = True
                    config_cart.visible = True
                    config_section.visible = True
                print(traceback.format_exc())

        def target():
            get_app().run_a_from_sync(helper, )

        est_price = update_estimates()
        if est_price > state['balance']:
            with main_ui:
                ui.notify(f"Insufficient balance. Need €{est_price:.2f}", type='negative')
        else:
            state['balance'] -= est_price
            save_user_state(session_id, state)
            with main_ui:
                online_states[0] += 1
                balance.set_text(f"Balance: {state['balance']:.2f}€ Running Queries: {online_states[0]}")

            Thread(target=target, daemon=True).start()
            with main_ui:
                online_states[0] -= 1
                balance.set_text(f"Balance: {get_user_state(session_id)['balance']:.2f}€")


    def show_history():
        with config_cart:
            for idx, entry in enumerate(state['research_history']):
                with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4"):
                    ui.label(entry['query']).classes('text-sm')
                    ui.button("Open").on_click(lambda _, i=idx: load_history(i))

    def reset():
        nonlocal processor_instance, results_card, followup_card, sr_button, r_button, r_text
        processor_instance["instance"] = None
        show_progress_indicators()
        with main_ui:
            config_cart.visible = False
            config_section.visible = False
            followup_card.visible = False
            results_card.visible = False
            r_button.visible = False
            r_text.set_text("Research Interface")
            sr_button.set_text("Start Research")
        start_search()
    # UI-Aufbau

    with ui.column().classes("w-full max-w-6xl mx-auto p-6 space-y-6") as main_ui:
        balance = ui.label(f"Balance: {state['balance']:.2f}€").classes("text-s font-semibold")

        config_cart = config_cart

        # --- Research Input UI Card ---
        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4") as research_card:
            r_text = ui.label("Research Interface").classes("text-3xl font-bold mb-4")

            # Query input section with auto-updating estimates
            query = ui.input("Research Query",
                                placeholder="Gib hier deine Forschungsfrage ein...",
                                value="")                     .classes("w-full min-h-[100px]")                     .on('change', lambda e: on_query_change()).style("color: var(--text-color)")

            # --- Action Buttons ---
            with ui.row().classes("mt-4"):
                sr_button =ui.button("Start Research", on_click=start_search)                         .classes("bg-blue-600 hover:bg-blue-700 py-3 rounded-lg")
                ui.button("toggle config",
                          on_click=lambda: setattr(config_section, 'visible', not config_section.visible) or show_progress_indicators()).style(
                    "color: var(--text-color)")
                r_button = ui.button("Start new Research",
                          on_click=reset).style(
                    "color: var(--text-color)")
        sr_button = sr_button
        r_button = r_button
        r_button.visible = False
        research_card = research_card

        # --- Options Cart / Configurations ---
        with ui.card_section().classes("w-full backdrop-blur-lg bg-white/10 hidden") as config_section:
            ui.separator()
            ui.label("Configuration Options").classes("text-xl font-semibold mt-4 mb-2")
            with ui.row():
                chunk_size_input = ui.number(label="Chunk Size",
                                             value=config['chunk_size'], format='%.0f', max=64_000, min=1000,
                                             step=100)                         .on('change', on_config_change).style("color: var(--text-color)")
                overlap_input = ui.number(label="Overlap",
                                          value=config['overlap'], format='%.0f', max=6400, min=100, step=50)                         .on('change', on_config_change).style("color: var(--text-color)")

            with ui.row():
                num_search_result_input = ui.number(label="Results per Query",
                                                    value=config['num_search_result_per_query'], format='%.0f',
                                                    min=1, max=100, step=1)                         .on('change', on_config_change).style("color: var(--text-color)")
                max_search_input = ui.number(label="Max Search Queries",
                                             value=config['max_search'], format='%.0f', min=1, max=100, step=1)                         .on('change', on_config_change).style("color: var(--text-color)")
                num_workers_input = ui.number(label="Number of Workers (leave empty for default)",
                                              value=0, format='%.0f', min=0, max=32, step=1)                         .on('change', on_config_change).style("color: var(--text-color)")
        config_section = config_section
        config_section.visible = False
        # --- Ergebnisse anzeigen ---
        with ui.card().classes("w-full backdrop-blur-lg p-4 bg-white/10") as results_card:
            ui.label("Research Results").classes("text-xl font-semibold mb-4")
            with ui.tabs() as tabs:
                ui.tab("Summary")
                ui.tab("References")
                ui.tab("SystemStates")
            with ui.tab_panels(tabs, value="Summary").classes("w-full").style("background-color: var(--background-color)"):
                with ui.tab_panel("Summary"):
                    summary_content = ui.markdown("").style("color : var(--text-color)")
                with ui.tab_panel("References"):
                    references_content = ui.markdown("").style("color : var(--text-color)")
                with ui.tab_panel("SystemStates"):
                    analysis_content = ui.markdown("").style("color : var(--text-color)")


        # Ergebnisse sichtbar machen, sobald sie vorliegen.
        results_card = results_card
        results_card.visible = False

        # --- Follow-Up Bereich mit mehrfachen Folgefragen und Suchparametern ---
        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4 hidden") as followup_card:
            pass

        # Zugriff auf followup_card (falls später benötigt)
        followup_card = followup_card
        followup_card.visible = False

        # --- Fortschrittsanzeige ---
        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4") as progress_card:
            with ui.row():
                ui.label("Research Progress").classes("text-xl font-semibold mb-4")
                query_length_label = ui.label("").classes("mt-6 hover:text-primary transition-colors duration-300")
                time_label = ui.label("Time: ...").classes("mt-6 hover:text-primary transition-colors duration-300")
                price_label = ui.label("Price: ...").classes(
                    "mt-6 hover:text-primary transition-colors duration-300")

            overall_progress = ui.linear_progress(0).classes("w-full mb-4")
            status_label = ui.label("Warte auf Start...").classes("text-base")
        # Wir merken uns progress_card, falls wir ihn zurücksetzen wollen.
        progress_card = progress_card

        query_length_label = query_length_label
        time_label = time_label
        price_label = price_label

        with ui.card().classes("w-full backdrop-blur-lg bg-white/10 p-4") as config_cart:
            # --- Process Code Section ---
            # --- Estimated Time and Price ---
            # ui.label("History").classes("text-xl font-semibold mt-4 mb-2")
            ui.label('Research History').classes('text-xl p-4')
            show_history()

        ui.button('Add Credits', on_click=lambda: balance_overlay(session_id)).props('icon=paid')
        ui.label('About TruthSeeker').classes(
            'mt-6 text-gray-500 hover:text-primary '
            'transition-colors duration-300'
        ).on('click', lambda: ui.navigate.to('/open-Seeker.about', new_tab=True))

        with ui.element('div').classes("w-full").style("white:100%; height:100%") as graph_ui:
            pass

        with ui.card().classes("w-full p-4").style("background-color: var(--background-color)"):
            ui.label("Private Session link (restore the session on a different device)")
            base_url = f'https://{os.getenv("HOSTNAME")}/gui/open-Seeker.seek' if not 'localhost' in os.getenv("HOSTNAME") else 'http://localhost:5000/gui/open-Seeker.seek'
            ui.label(f"{base_url}?session_id={session_id}").style("white:100%")
            ui.label("Changes each time!")

        graph_ui = graph_ui
        graph_ui.visible = False
    main_ui = main_ui
    main_ui.visible = False

    # --- Hilfsfunktionen ---
    def validate_inputs() -> bool:
        if not query.value.strip():
            with main_ui:
                ui.notify("Bitte gib eine Forschungsfrage ein.", type="warning")
            return False
        return True

    def reset_interface():
        nonlocal overall_progress, status_label, results_card, followup_card
        overall_progress.value = 0
        with main_ui:
            status_label.set_text("Research startet...")
        # Ergebnisse und Follow-Up Bereich verstecken
        results_card.visible = False
        followup_card.visible = False
        graph_ui.visible = False

    def show_progress_indicators():
        nonlocal progress_card
        progress_card.visible = True

    def update_results(data: dict, save=True):
        nonlocal summary_content, analysis_content, references_content, results_card,                followup_card,graph_ui, r_button, r_text, sr_button
        with main_ui:
            r_button.visible = True
            r_text.set_text("Add to current Results or press 'Start new Research'")
            sr_button.set_text("Add to current Results")
        # Handle papers (1-to-1 case)
        papers = data.get("papers", [])
        if not isinstance(papers, list):
            papers = [papers]

        # Get insights
        insights = data.get("insights", [])

        if save:
            history_entry = data.copy()
            history_entry['papers'] = [paper.model_dump_json() for paper in papers]
            if processor_instance is not None and processor_instance['instance'] is not None:
                history_entry["mam_name"] = processor_instance['instance'].mem_name
                history_entry["query"] = processor_instance['instance'].query

                history_entry["processor_memory"] = processor_instance['instance'].tools.get_memory(

                ).save_memory(history_entry["mam_name"], None)
            state['research_history'].append(history_entry)
            save_user_state(session_id, state)
        else:
            papers = [Paper(**json.loads(paper)) for paper in papers]
        create_followup_section(processor_instance, followup_card, session_id, balance)
        with main_ui:
            progress_card.visible = False
            # Build summary from insights
            summaries = []
            for insight in insights:
                if 'result' in insight and 'summary' in insight['result']:
                    if isinstance(insight['result']['summary'], str):
                        # print(insight['result']['summary'], "NEXT", json.loads(insight['result']['summary'][:-1]),"NEXT22",  type(json.loads(insight['result']['summary'][:-1])))
                        insight['result']['summary'] = json.loads(insight['result']['summary'][:-1])
                    main_summary = insight['result']['summary'].get('main_summary', '')
                    if main_summary:
                        summaries.append(main_summary)
            summary_text = "

".join(summaries) if summaries else "No summary available." summary_content.set_content(f"# Research Summary

{summary_text}")

            # Analysis section (unchanged if processor details haven't changed)
            if processor_instance["instance"] is not None:
                inst = processor_instance["instance"]
                analysis_md = (
                    f"# Analysis

" f"- query: {inst.query} " f"- chunk_size: {inst.chunk_size} " f"- overlap: {inst.overlap} " f"- max_workers: {inst.max_workers} " f"- num_search_result_per_query: {inst.nsrpq} " f"- max_search: {inst.max_search} " f"- download_dir: {inst.download_dir} " f"- mem_name: {inst.mem_name} " f"- current_session: {inst.current_session} " f"- all_ref_papers: {inst.all_ref_papers} " f"- all_texts_len: {inst.all_texts_len} " f"- final_texts_len: {inst.f_texts_len} " f"- num_workers: {inst.num_workers}" ) analysis_content.set_content(analysis_md)

            # References and Insights section
            references_md = "# References

" # Add papers references_md += " ".join( f"- ({i}) {getattr(paper, 'title', 'Unknown Title')}})" for i, paper in enumerate(papers) )

            # Add detailed insights
            references_md += "
Insights

" for i, insight in enumerate(insights): print(insight) result = insight.get('result', {}) summary = result.get('summary', {})

                if isinstance(summary, str):
                    summary = json.loads(summary)

                # Main summary
                references_md += f"
Insight

" references_md += f"### Main Summary {summary.get('main_summary', 'No summary available.')} "

                # Concept Analysis
                concept_analysis = summary.get('concept_analysis', {})
                if concept_analysis:
                    references_md += "
Concept Analysis

" references_md += "#### Key Concepts - " + " - ".join( concept_analysis.get('key_concepts', [])) + " " references_md += "

Relationships
  • " + "
  • ".join( concept_analysis.get('relationships', [])) + " " references_md += "
Importance Hierarchy
  • " + "
  • ".join( concept_analysis.get('importance_hierarchy', [])) + " "

                # Topic Insights
                topic_insights = summary.get('topic_insights', {})
                if topic_insights:
                    references_md += "
    
    Topic Insights

    " references_md += "#### Primary Topics - " + " - ".join( topic_insights.get('primary_topics', [])) + " " references_md += "

    Cross References
    • " + "
    • ".join( topic_insights.get('cross_references', [])) + " " references_md += "
    Knowledge Gaps
    • " + "
    • ".join( topic_insights.get('knowledge_gaps', [])) + " "

              # Relevance Assessment
              relevance = summary.get('relevance_assessment', {})
              if relevance:
                  references_md += "
      
      Relevance Assessment

      " references_md += f"- Query Alignment: {relevance.get('query_alignment', 'N/A')} " references_md += f"- Confidence Score: {relevance.get('confidence_score', 'N/A')} " references_md += f"- Coverage Analysis: {relevance.get('coverage_analysis', 'N/A')} "

          references_content.set_content(references_md)
      
          # nx concpts graph
          if processor_instance["instance"] is not None:
              create_graph_tab(
                  processor_instance,
                  graph_ui,main_ui
              )
      
          # Show results and followup cards
          results_card.visible = True
          followup_card.visible = True
          graph_ui.visible = True
      

      def load_history(index: int): entry = state['research_history'][index] if processor_instance is not None and processor_instance['instance'] is not None:

          processor_instance["instance"].mem_name = entry["mam_name"]
          processor_instance['instance'].query = entry["query"]
      
          pass
      else:
          processor = Processor(entry["query"], tools=get_tools(), **config)
          # Setze den Callback so, dass Updates in der GUI angezeigt werden
          processor.callback = update_status
          processor.mem_name = entry["mam_name"]
          processor_instance["instance"] = processor
      
      processor_instance["instance"].tools.get_memory().load_memory(entry["mam_name"], entry["processor_memory"])
      processor_instance["instance"].mem_name = entry["mam_name"]
      update_results(entry, save=False)
      

    return helpr

--- Stripe Integration ---

def regiser_stripe_integration(is_scc=True): def stripe_callback(request: Request):

    sid = request.row.query_params.get('session_id') if hasattr(request, 'row') else request.query_params.get('session_id')
    state = get_user_state(sid)

    if state['payment_id'] == '':
        with ui.card().classes("w-full items-center").style("background-color: var(--background-color)"):
            ui.label(f"No payment id!").classes("text-lg font-bold")
            ui.button(
                "Start Research",
                on_click=lambda: ui.navigate.to("/open-Seeker.seek?session_id="+sid)
            ).classes(
                "w-full px-6 py-4 text-lg font-bold "
                "bg-primary hover:bg-primary-dark "
                "transform hover:-translate-y-0.5 "
                "transition-all duration-300 ease-in-out "
                "rounded-xl shadow-lg animate-slideUp"
            )
        return

    try:
        session_data = stripe.checkout.Session.retrieve(state['payment_id'])
    except Exception as e:
        with ui.card().classes("w-full items-center").style("background-color: var(--background-color)"):
            ui.label(f"No Transactions Details !{e}").classes("text-lg font-bold")
            ui.button(
                "Start Research",
                on_click=lambda: ui.navigate.to("/open-Seeker.seek")
            ).classes(
                "w-full px-6 py-4 text-lg font-bold "
                "bg-primary hover:bg-primary-dark "
                "transform hover:-translate-y-0.5 "
                "transition-all duration-300 ease-in-out "
                "rounded-xl shadow-lg animate-slideUp"
            )
            return
    with ui.card().classes("w-full items-center").style("background-color: var(--background-color)"):
        if is_scc and state['payment_id'] != '' and session_data.payment_status == 'paid':
            state = get_user_state(sid)
            amount = session_data.amount_total / 100  # Convert cents to euros
            state['balance'] += amount
            state['payment_id'] = ''
            save_user_state(sid, state)

        # ui.navigate.to(f'/session?session={session}')
            ui.label(f"Transaction Complete - New balance :{state['balance']}").classes("text-lg font-bold")
            with ui.card().classes("w-full p-4").style("background-color: var(--background-color)"):
                ui.label("Private Session link (restore the session on a different device)")
                base_url = f'https://{os.getenv("HOSTNAME")}/gui/open-Seeker.seek' if not 'localhost' in os.getenv("HOSTNAME")else 'http://localhost:5000/gui/open-Seeker.seek'
                ui.label(f"{base_url}?session_id={sid}").style("white:100%")
                ui.label("Changes each time!")
        else:
            ui.label(f"Transaction Error! {session_data}, {dir(session_data)}").classes("text-lg font-bold")
        ui.button(
            "Start Research",
            on_click=lambda: ui.navigate.to("/open-Seeker.seek")
        ).classes(
            "w-full px-6 py-4 text-lg font-bold "
            "bg-primary hover:bg-primary-dark "
            "transform hover:-translate-y-0.5 "
            "transition-all duration-300 ease-in-out "
            "rounded-xl shadow-lg animate-slideUp"
        )


return stripe_callback

def handle_stripe_payment(amount: float, session_id): base_url = f'https://{os.getenv("HOSTNAME")}/gui/open-Seeker.stripe' if not 'localhost' in os.getenv("HOSTNAME") else 'http://localhost:5000/gui/open-Seeker.stripe' session = stripe.checkout.Session.create( payment_method_types=['card', "link", ], line_items=[{ 'price_data': { 'currency': 'eur', 'product_data': {'name': 'Research Credits'}, 'unit_amount': int(amount * 100), }, 'quantity': 1, }], automatic_tax={"enabled": True}, mode='payment', success_url=f'{base_url}?session_id={session_id}', cancel_url=f'{base_url}.error' ) state = get_user_state(session_id) state['payment_id'] = session.id save_user_state(session_id, state) ui.navigate.to(session.url, new_tab=True)

--- UI Components ---

def balance_overlay(session_id): with ui.dialog().classes('w-full max-w-md bg-white/20 backdrop-blur-lg rounded-xl') as dialog: with ui.card().classes('w-full p-6 space-y-4').style("background-color: var(--background-color)"): ui.label('Add Research Credits').classes('text-2xl font-bold') amount = ui.number('Amount (€) min 2', value=5, format='%.2f', min=2, max=9999, step=1).classes('w-full') with ui.row().classes('w-full justify-between'): ui.button('Cancel', on_click=dialog.close).props('flat') ui.button('Purchase', on_click=lambda: handle_stripe_payment(amount.value, session_id)) return dialog

def create_ui(processor): # ui_instance = register_nicegui("open-Seeker", create_landing_page , additional=""" """, show=False) register_nicegui("open-Seeker.demo", create_video_demo, additional=""" """, show=False)

newui

cleanup_module(app)

Cleanup resources when the module is unloaded

Source code in toolboxv2/mods/TruthSeeker/newui.py
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
@export(mod_name=MOD_NAME, version=version, exit_f=True)
def cleanup_module(app: App):
    """Cleanup resources when the module is unloaded"""
    # Clean up any temp files or resources
    import glob
    import shutil

    # Remove temporary PDF directories
    for pdf_dir in glob.glob("pdfs_*"):
        try:
            shutil.rmtree(pdf_dir)
        except Exception as e:
            print(f"Error removing directory {pdf_dir}: {str(e)}")

    # Clear any SSE queues
    if hasattr(app, 'sse_queues'):
        app.sse_queues = {}

    if hasattr(app, 'payment_queues'):
        app.payment_queues = {}

    return Result.ok(info="ArXivPDFProcessor UI cleaned up")
create_payment(app, data) async

Create a Stripe payment session

Source code in toolboxv2/mods/TruthSeeker/newui.py
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
@export(mod_name=MOD_NAME, api=True, version=version)
async def create_payment(app: App, data):
    """Create a Stripe payment session"""
    amount = data.get("amount")
    session_id = data.get("session_id")

    if amount < 2:
        return Result.default_user_error(info="Minimum donation amount is €2")

    try:
        # Create a Stripe Checkout Session
        base_url = f"https://{os.getenv('HOSTNAME', 'localhost:5000')}"
        success_url = f"{base_url}/api/{MOD_NAME}/payment_success?session_id={session_id}"
        cancel_url = f"{base_url}/api/{MOD_NAME}/payment_cancel?session_id={session_id}"
        stripe = __import__('stripe')
        stripe.api_key = os.getenv('STRIPE_SECRET_KEY', 'sk_test_YourSecretKey')

        stripe_session = stripe.checkout.Session.create(
            payment_method_types=['card', 'link'],
            line_items=[{
                'price_data': {
                    'currency': 'eur',
                    'product_data': {'name': 'Research Credits'},
                    'unit_amount': int(amount * 100),
                },
                'quantity': 1,
            }],
            automatic_tax={"enabled": True},
            mode='payment',
            success_url=success_url,
            cancel_url=cancel_url
        )

        # Store the payment info
        if not hasattr(app, 'payment_info'):
            app.payment_info = {}

        # Initialize payment_queues if not already done
        if not hasattr(app, 'payment_queues'):
            app.payment_queues = {}

        # Create a queue for this payment
        app.payment_queues[session_id] = asyncio.Queue()

        app.payment_info[session_id] = {
            'payment_id': stripe_session.id,
            'amount': amount,
            'status': 'pending'
        }

        return Result.ok(data={"url": stripe_session.url})
    except Exception as e:
        return Result.default_internal_error(info=f"Error creating payment: {str(e)}")
estimate_processing(data) async

Estimate processing time and cost for a given query

Source code in toolboxv2/mods/TruthSeeker/newui.py
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
@export(mod_name=MOD_NAME, api=True, version=version)
async def estimate_processing(data):
    """Estimate processing time and cost for a given query"""
    # Use the static method to estimate metrics
    query, max_search, num_search_result_per_query= data.get("query", ""), data.get("max_search",4), data.get("num_search_result_per_query",6)
    estimated_time, estimated_price = ArXivPDFProcessor.estimate_processing_metrics(
        query_length=len(query),
        max_search=max_search,
        num_search_result_per_query=num_search_result_per_query,
        chunk_size=1_000_000,
        overlap=2_000,
        num_workers=None
    )

    return Result.ok(data={
        "time": estimated_time,
        "price": estimated_price
    })
follow_up_query(app, data) async

Ask a follow-up question about the research

Source code in toolboxv2/mods/TruthSeeker/newui.py
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
@export(mod_name=MOD_NAME, api=True, version=version)
async def follow_up_query(app: App, data):
    """Ask a follow-up question about the research"""
    research_id = data.get("research_id")
    query = data.get("query")

    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    research_process = app.research_processes[research_id]

    if research_process['status'] != 'complete':
        return Result.default_user_error(info="Research is not complete")

    processor = research_process['processor']
    if not processor:
        return Result.default_user_error(info="Processor not available")

    try:
        # Use the extra_query method to ask follow-up questions
        result = await processor.extra_query(query)

        return Result.ok(data={"answer": result['response'] if result and 'response' in result else "No response"})
    except Exception as e:
        return Result.default_internal_error(info=f"Error processing follow-up query: {str(e)}")
initialize_module(app)

Initialize the module and register UI with CloudM

Source code in toolboxv2/mods/TruthSeeker/newui.py
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
@export(mod_name=MOD_NAME, version=version, initial=True)
def initialize_module(app: App):
    """Initialize the module and register UI with CloudM"""
    # Register the UI with CloudM
    app.run_any(("CloudM", "add_ui"),
                name="TruthSeeker",
                title="TruthSeeker Research",
                path=f"/api/{MOD_NAME}/get_main_ui",
                description="AI Research Assistant"
                )

    # Initialize SSE message queues
    if not hasattr(app, 'sse_queues'):
        app.sse_queues = {}
    print("TruthSeeker online")
    return Result.ok(info="ArXivPDFProcessor UI initialized")
payment_cancel(app, session_id, request_as_kwarg=True, request=None) async

Handle cancelled payment

Source code in toolboxv2/mods/TruthSeeker/newui.py
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
@export(mod_name=MOD_NAME, api=True, version=version)
async def payment_cancel(app: App, session_id: str, request_as_kwarg=True, request=None):
    """Handle cancelled payment"""
    if hasattr(app, 'payment_info') and session_id in app.payment_info:
        app.payment_info[session_id]['status'] = 'cancelled'

        # Notify SSE clients about payment cancellation
        if hasattr(app, 'payment_queues') and session_id in app.payment_queues:
            await app.payment_queues[session_id].put({
                "status": "cancelled"
            })

    return Result.html(app.web_context() + """
    <div style="text-align: center; padding: 50px;">
        <h2>Payment Cancelled</h2>
        <p>Your payment was cancelled.</p>
        <script>
            setTimeout(function() {
                window.close();
            }, 3000);
        </script>
    </div>
    """)
payment_stream(app, session_id) async

SSE stream endpoint for payment status updates

Source code in toolboxv2/mods/TruthSeeker/newui.py
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
@export(mod_name=MOD_NAME, api=True, version=version)
async def payment_stream(app: App, session_id: str):
    """SSE stream endpoint for payment status updates"""
    if not hasattr(app, 'payment_queues'):
        app.payment_queues = {}

    # Create a message queue for this session_id if it doesn't exist
    if session_id not in app.payment_queues:
        app.payment_queues[session_id] = asyncio.Queue()

    async def generate():
        try:
            # Stream payment updates
            while True:
                try:
                    # Wait for a payment update with a timeout
                    payment_data = await asyncio.wait_for(app.payment_queues[session_id].get(), timeout=30)
                    yield f"event: payment_update\ndata: {json.dumps(payment_data)}\n\n"

                    # If the payment is complete or cancelled, exit the loop
                    if payment_data.get('status') in ['completed', 'cancelled']:
                        break
                except TimeoutError:
                    # Send a keep-alive comment to prevent connection timeout
                    yield ":\n\n"
        finally:
            # Clean up resources when the client disconnects
            if session_id in app.payment_queues:
                # Keep the queue for other potential clients
                pass

    return Result.stream(generate())
payment_success(app, session_id, request_as_kwarg=True, request=None) async

Handle successful payment

Source code in toolboxv2/mods/TruthSeeker/newui.py
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
@export(mod_name=MOD_NAME, api=True, version=version)
async def payment_success(app: App, session_id: str, request_as_kwarg=True, request=None):
    """Handle successful payment"""
    if not hasattr(app, 'payment_info') or session_id not in app.payment_info:
        return Result.html(app.web_context() + """
        <div style="text-align: center; padding: 50px;">
            <h2>Payment Session Not Found</h2>
            <p>Return to the main page to continue.</p>
            <a href="/" style="display: inline-block; margin-top: 20px; padding: 10px 20px; background-color: #4F46E5; color: white; text-decoration: none; border-radius: 5px;">Return to Home</a>
        </div>
        """)

    payment_info = app.payment_info[session_id]

    try:
        # Verify the payment with Stripe
        stripe = __import__('stripe')
        stripe.api_key = os.getenv('STRIPE_SECRET_KEY', 'sk_test_YourSecretKey')

        stripe_session = stripe.checkout.Session.retrieve(payment_info['payment_id'])

        if stripe_session.payment_status == 'paid':
            payment_info['status'] = 'completed'

            # Notify SSE clients about payment completion
            if hasattr(app, 'payment_queues') and session_id in app.payment_queues:
                await app.payment_queues[session_id].put({
                    "status": "completed",
                    "amount": payment_info['amount']
                })

            return Result.html(app.web_context() + """
            <div style="text-align: center; padding: 50px;">
                <h2>Thank You for Your Support!</h2>
                <p>Your payment was successful. You can now close this window and continue with your research.</p>
                <script>
                    setTimeout(function() {
                        window.close();
                    }, 5000);
                </script>
            </div>
            """)
        else:
            return Result.html(app.web_context() + """
            <div style="text-align: center; padding: 50px;">
                <h2>Payment Not Completed</h2>
                <p>Your payment has not been completed. Please try again.</p>
                <button onclick="window.close()">Close Window</button>
            </div>
            """)
    except Exception as e:
        return Result.html(app.web_context() + f"""
        <div style="text-align: center; padding: 50px;">
            <h2>Error Processing Payment</h2>
            <p>There was an error processing your payment: {str(e)}</p>
            <button onclick="window.close()">Close Window</button>
        </div>
        """)
research_results(app, research_id) async

Get the results of a completed research process

Source code in toolboxv2/mods/TruthSeeker/newui.py
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
@export(mod_name=MOD_NAME, api=True, version=version)
async def research_results(app: App, research_id: str):
    """Get the results of a completed research process"""
    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    research_process = app.research_processes[research_id]

    if research_process['status'] != 'complete':
        return Result.default_user_error(info="Research is not complete")

    return Result.ok(data=research_process['results'])
research_status(app, research_id) async

Get the status of a research process

Source code in toolboxv2/mods/TruthSeeker/newui.py
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
@export(mod_name=MOD_NAME, api=True, version=version)
async def research_status(app: App, research_id: str):
    """Get the status of a research process"""
    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    research_process = app.research_processes[research_id]

    return Result.ok(data={
        "status": research_process['status'],
        "progress": research_process['progress'],
        "step": research_process['step'],
        "info": research_process['info']
    })
start_research(app, data) async

Start a new research process

Source code in toolboxv2/mods/TruthSeeker/newui.py
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
@export(mod_name=MOD_NAME, api=True, version=version)
async def start_research(app: App, data):
    """Start a new research process"""
    # Get data from the request
    query = data.get("query")
    session_id = data.get("session_id")
    max_search = data.get("max_search", 4)
    num_search_result_per_query = data.get("num_search_result_per_query", 4)

    # Get the tools module
    tools = get_app("ArXivPDFProcessor").get_mod("isaa")
    if not hasattr(tools, 'initialized') or not tools.initialized:
        tools.init_isaa(build=True)

    # Generate a unique research_id
    research_id = str(uuid.uuid4())

    # Store the research information in a global dictionary
    if not hasattr(app, 'research_processes'):
        app.research_processes = {}

    # Initialize SSE queues if not already done
    if not hasattr(app, 'sse_queues'):
        app.sse_queues = {}

    # Create a queue for this research process
    app.sse_queues[research_id] = asyncio.Queue()

    # Create a processor with callback for status updates
    app.research_processes[research_id] = {
        'status': 'initializing',
        'progress': 0.0,
        'step': 'Initializing',
        'info': '',
        'query': query,
        'session_id': session_id,
        'processor': None,
        'results': None,
        'stop_requested': False
    }

    # Define the callback function that sends updates to the SSE queue
    def status_callback(status_data):
        if research_id in app.research_processes:
            process = app.research_processes[research_id]
            process['status'] = 'processing'
            process['progress'] = status_data.get('progress', 0.0)
            process['step'] = status_data.get('step', '')
            process['info'] = status_data.get('info', '')

            # Put the status update in the SSE queue
            status_update = {
                "status": process['status'],
                "progress": process['progress'],
                "step": process['step'],
                "info": process['info']
            }

            if research_id in app.sse_queues:
                asyncio.create_task(app.sse_queues[research_id].put(status_update))

    # Create the processor
    processor = ArXivPDFProcessor(
        query=query,
        tools=tools,
        chunk_size=1_000_000,
        overlap=2_000,
        max_search=max_search,
        num_search_result_per_query=num_search_result_per_query,
        download_dir=f"pdfs_{research_id}",
        callback=status_callback
    )

    app.research_processes[research_id]['processor'] = processor

    # Process in the background
    async def process_in_background():
        try:
            # Check if stop was requested before starting
            if app.research_processes[research_id]['stop_requested']:
                app.research_processes[research_id]['status'] = 'stopped'
                if research_id in app.sse_queues:
                    await app.sse_queues[research_id].put({
                        "status": "stopped",
                        "progress": 0,
                        "step": "Research stopped",
                        "info": ""
                    })
                return

            # Start processing
            papers, insights = await processor.process()

            # Check if stop was requested during processing
            if app.research_processes[research_id]['stop_requested']:
                app.research_processes[research_id]['status'] = 'stopped'
                if research_id in app.sse_queues:
                    await app.sse_queues[research_id].put({
                        "status": "stopped",
                        "progress": 1,
                        "step": "Research stopped",
                        "info": ""
                    })
                return

            # Store results
            app.research_processes[research_id]['results'] = {
                'papers': papers,
                'insights': insights['response'] if insights and 'response' in insights else None
            }
            app.research_processes[research_id]['status'] = 'complete'

            # Send final status update
            if research_id in app.sse_queues:
                await app.sse_queues[research_id].put({
                    "status": "complete",
                    "progress": 1,
                    "step": "Research complete",
                    "info": f"Found {len(papers)} papers"
                })

        except Exception as e:
            app.research_processes[research_id]['status'] = 'error'
            app.research_processes[research_id]['info'] = str(e)

            # Send error status
            if research_id in app.sse_queues:
                await app.sse_queues[research_id].put({
                    "status": "error",
                    "progress": 0,
                    "step": "Error",
                    "info": str(e)
                })

            print(f"Error in research process {research_id}: {str(e)}")

    # Start the background task
    asyncio.create_task(process_in_background())

    return Result.ok(data={"research_id": research_id})
status_stream(app, research_id) async

SSE stream endpoint for research status updates

Source code in toolboxv2/mods/TruthSeeker/newui.py
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
@export(mod_name=MOD_NAME, api=True, version=version)
async def status_stream(app: App, research_id: str):
    """SSE stream endpoint for research status updates"""
    if not hasattr(app, 'sse_queues'):
        app.sse_queues = {}

    # Create a message queue for this research_id if it doesn't exist
    if research_id not in app.sse_queues:
        app.sse_queues[research_id] = asyncio.Queue()

    async def generate():
        # Send initial status
        if hasattr(app, 'research_processes') and research_id in app.research_processes:
            process = app.research_processes[research_id]
            initial_status = {
                "status": process['status'],
                "progress": process['progress'],
                "step": process['step'],
                "info": process['info']
            }
            yield f"event: status_update\ndata: {json.dumps(initial_status)}\n\n"

        try:
            # Stream status updates
            while True:
                try:
                    # Wait for a new status update with a timeout
                    status_data = await asyncio.wait_for(app.sse_queues[research_id].get(), timeout=30)
                    yield f"event: status_update\ndata: {json.dumps(status_data)}\n\n"

                    # If the research is complete or there was an error, exit the loop
                    if status_data.get('status') in ['complete', 'error', 'stopped']:
                        break
                except TimeoutError:
                    # Send a keep-alive comment to prevent connection timeout
                    yield ":\n\n"
        finally:
            # Clean up resources when the client disconnects
            if research_id in app.sse_queues:
                # Keep the queue for other potential clients
                pass

    return Result.stream(generate())
stop_research(app, data) async

Stop a research process

Source code in toolboxv2/mods/TruthSeeker/newui.py
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
@export(mod_name=MOD_NAME, api=True, version=version)
async def stop_research(app: App, data):
    """Stop a research process"""
    research_id = data.get("research_id")
    if not hasattr(app, 'research_processes') or research_id not in app.research_processes:
        return Result.default_user_error(info="Research process not found")

    app.research_processes[research_id]['stop_requested'] = True

    # Send stopped status to SSE clients
    if hasattr(app, 'sse_queues') and research_id in app.sse_queues:
        await app.sse_queues[research_id].put({
            "status": "stopped",
            "progress": app.research_processes[research_id]['progress'],
            "step": "Stopping research",
            "info": ""
        })

    return Result.ok(data={"status": "stop_requested"})

tests

TestTruthSeeker

Bases: TestCase

Source code in toolboxv2/mods/TruthSeeker/tests.py
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
class TestTruthSeeker(unittest.TestCase):
    def setUp(self):
        # Mock the App class
        self.mock_app = Mock()
        self.mock_app.get_mod.return_value = Mock()

        # Setup mock for run_any that returns iterable dict
        self.mock_app.run_any.return_value = {
            "1": {"name": "template1"},
            "2": {"name": "template2"}
        }

        # Mock RequestSession
        self.mock_request = Mock()
        self.mock_request.json = AsyncMock()

    @patch('os.path.join')
    @patch('builtins.open', create=True)
    def test_start_initialization(self, mock_open, mock_join):
        """Test the start function initializes correctly"""
        # Setup mock file handling
        mock_file = Mock()
        mock_file.read.return_value = "test content"
        mock_open.return_value.__enter__.return_value = mock_file

        # Call start function
        start(self.mock_app)

        # Verify app initialization calls
        self.mock_app.get_mod.assert_called_with("CodeVerification")
        self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker")
        self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker-promo")

    @async_test
    async def test_codes_valid_request(self):
        """Test the codes function with valid input"""
        # Mock request data
        test_data = {
            "query": "test query",
            "depth": "Q",
            "promoCode": "PROMO15",
            "ontimeCode": "TEST123"
        }
        self.mock_request.json.return_value = test_data

        # Mock code verification
        self.mock_app.run_any.return_value = {
            "template_name": "Promo15",
            "usage_type": "one_time"
        }

        result = await codes(self.mock_app, self.mock_request)

        self.assertTrue(result['valid'])
        self.assertIn('ontimeKey', result)
        self.assertIn('ppc', result)

    @async_test
    async def test_codes_invalid_promo(self):
        """Test the codes function with invalid promo code"""
        test_data = {
            "query": "test query",
            "depth": "I",
            "promoCode": "INVALID",
            "ontimeCode": "TEST123"
        }
        self.mock_request.json.return_value = test_data

        # Mock invalid promo code verification
        self.mock_app.run_any.return_value = None

        result = await codes(self.mock_app, self.mock_request)

        self.assertIn('ppc', result)
        self.assertTrue(result['ppc']['price'] > 0)

    @async_test
    async def test_process_valid_request(self):
        """Test the process function with valid input"""
        test_data = {
            "query": "test query",
            "depth": "Q",
            "ontimeKey": "VALID_KEY",
            "email": "test@example.com"
        }
        self.mock_request.json.return_value = test_data

        # Mock valid key verification
        self.mock_app.run_any.return_value = {
            "template_name": "PROCESS",
            "usage_type": "timed",
            "uses_count": 1
        }

        # Mock ArXivPDFProcessor
        with patch('toolboxv2.mods.TruthSeeker.module.ArXivPDFProcessor') as mock_processor:
            mock_insights = MagicMock()
            mock_insights.is_true = "True"
            mock_insights.summary = "Test summary"
            mock_insights.key_point = "Point1>\n\n<Point2"

            mock_processor.return_value.process.return_value = ([], mock_insights)

            result = await process(self.mock_app, self.mock_request)

            self.assertEqual(result['is_true'], "True")
            self.assertEqual(result['summary'], "Test summary")

    @async_test
    async def test_process_invalid_key(self):
        """Test the process function with invalid key"""
        test_data = {
            "query": "test query",
            "depth": "Q",
            "ontimeKey": "INVALID_KEY",
            "email": "test@example.com"
        }
        self.mock_request.json.return_value = test_data

        # Mock invalid key verification
        self.mock_app.run_any.return_value = None

        result = await process(self.mock_app, self.mock_request)

        self.assertEqual(result['summary'], "INVALID QUERY")
        self.assertEqual(result['insights'], [])
        self.assertEqual(result['papers'], [])

    def test_byCode_functionality(self):
        """Test the byCode function"""
        test_request = Mock()
        test_request.json.return_value = ["payKey", "codeClass", "ontimeKey"]

        result = byCode(self.mock_app, test_request)

        self.assertEqual(result, {'code': 'code'})
test_byCode_functionality()

Test the byCode function

Source code in toolboxv2/mods/TruthSeeker/tests.py
337
338
339
340
341
342
343
344
def test_byCode_functionality(self):
    """Test the byCode function"""
    test_request = Mock()
    test_request.json.return_value = ["payKey", "codeClass", "ontimeKey"]

    result = byCode(self.mock_app, test_request)

    self.assertEqual(result, {'code': 'code'})
test_codes_invalid_promo() async

Test the codes function with invalid promo code

Source code in toolboxv2/mods/TruthSeeker/tests.py
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
@async_test
async def test_codes_invalid_promo(self):
    """Test the codes function with invalid promo code"""
    test_data = {
        "query": "test query",
        "depth": "I",
        "promoCode": "INVALID",
        "ontimeCode": "TEST123"
    }
    self.mock_request.json.return_value = test_data

    # Mock invalid promo code verification
    self.mock_app.run_any.return_value = None

    result = await codes(self.mock_app, self.mock_request)

    self.assertIn('ppc', result)
    self.assertTrue(result['ppc']['price'] > 0)
test_codes_valid_request() async

Test the codes function with valid input

Source code in toolboxv2/mods/TruthSeeker/tests.py
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
@async_test
async def test_codes_valid_request(self):
    """Test the codes function with valid input"""
    # Mock request data
    test_data = {
        "query": "test query",
        "depth": "Q",
        "promoCode": "PROMO15",
        "ontimeCode": "TEST123"
    }
    self.mock_request.json.return_value = test_data

    # Mock code verification
    self.mock_app.run_any.return_value = {
        "template_name": "Promo15",
        "usage_type": "one_time"
    }

    result = await codes(self.mock_app, self.mock_request)

    self.assertTrue(result['valid'])
    self.assertIn('ontimeKey', result)
    self.assertIn('ppc', result)
test_process_invalid_key() async

Test the process function with invalid key

Source code in toolboxv2/mods/TruthSeeker/tests.py
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
@async_test
async def test_process_invalid_key(self):
    """Test the process function with invalid key"""
    test_data = {
        "query": "test query",
        "depth": "Q",
        "ontimeKey": "INVALID_KEY",
        "email": "test@example.com"
    }
    self.mock_request.json.return_value = test_data

    # Mock invalid key verification
    self.mock_app.run_any.return_value = None

    result = await process(self.mock_app, self.mock_request)

    self.assertEqual(result['summary'], "INVALID QUERY")
    self.assertEqual(result['insights'], [])
    self.assertEqual(result['papers'], [])
test_process_valid_request() async

Test the process function with valid input

Source code in toolboxv2/mods/TruthSeeker/tests.py
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
@async_test
async def test_process_valid_request(self):
    """Test the process function with valid input"""
    test_data = {
        "query": "test query",
        "depth": "Q",
        "ontimeKey": "VALID_KEY",
        "email": "test@example.com"
    }
    self.mock_request.json.return_value = test_data

    # Mock valid key verification
    self.mock_app.run_any.return_value = {
        "template_name": "PROCESS",
        "usage_type": "timed",
        "uses_count": 1
    }

    # Mock ArXivPDFProcessor
    with patch('toolboxv2.mods.TruthSeeker.module.ArXivPDFProcessor') as mock_processor:
        mock_insights = MagicMock()
        mock_insights.is_true = "True"
        mock_insights.summary = "Test summary"
        mock_insights.key_point = "Point1>\n\n<Point2"

        mock_processor.return_value.process.return_value = ([], mock_insights)

        result = await process(self.mock_app, self.mock_request)

        self.assertEqual(result['is_true'], "True")
        self.assertEqual(result['summary'], "Test summary")
test_start_initialization(mock_open, mock_join)

Test the start function initializes correctly

Source code in toolboxv2/mods/TruthSeeker/tests.py
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
@patch('os.path.join')
@patch('builtins.open', create=True)
def test_start_initialization(self, mock_open, mock_join):
    """Test the start function initializes correctly"""
    # Setup mock file handling
    mock_file = Mock()
    mock_file.read.return_value = "test content"
    mock_open.return_value.__enter__.return_value = mock_file

    # Call start function
    start(self.mock_app)

    # Verify app initialization calls
    self.mock_app.get_mod.assert_called_with("CodeVerification")
    self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker")
    self.mock_app.run_any.assert_any_call(("CodeVerification", "init_scope"), scope="TruthSeeker-promo")
run_all_tests()

Run all test classes

Source code in toolboxv2/mods/TruthSeeker/tests.py
393
394
395
396
@default_test
def run_all_tests():
    """Run all test classes"""
    return run_test_suite()
run_arxiv_processor_tests(test_name=None)

Run TestArXivPDFProcessor tests

Source code in toolboxv2/mods/TruthSeeker/tests.py
380
381
382
def run_arxiv_processor_tests(test_name=None):
    """Run TestArXivPDFProcessor tests"""
    return run_test_suite(TestArXivPDFProcessor, test_name)
run_pdf_downloader_tests(test_name=None)

Run TestRobustPDFDownloader tests

Source code in toolboxv2/mods/TruthSeeker/tests.py
375
376
377
def run_pdf_downloader_tests(test_name=None):
    """Run TestRobustPDFDownloader tests"""
    return run_test_suite(TestRobustPDFDownloader, test_name)
run_specific_test(test_class, test_name)

Run a specific test from a test class

Source code in toolboxv2/mods/TruthSeeker/tests.py
389
390
391
def run_specific_test(test_class, test_name):
    """Run a specific test from a test class"""
    return run_test_suite(test_class, test_name)
run_test_suite(test_class=None, test_name=None, verbosity=2)

Run specific test class or test case.

Parameters:

Name Type Description Default
test_class

The test class to run (optional)

None
test_name

Specific test method name to run (optional)

None
verbosity

Output detail level (default=2)

2

Returns:

Type Description

TestResult object

Source code in toolboxv2/mods/TruthSeeker/tests.py
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
def run_test_suite(test_class=None, test_name=None, verbosity=2):
    """
    Run specific test class or test case.

    Args:
        test_class: The test class to run (optional)
        test_name: Specific test method name to run (optional)
        verbosity: Output detail level (default=2)

    Returns:
        TestResult object
    """
    loader = unittest.TestLoader()
    suite = unittest.TestSuite()

    if test_class and test_name:
        # Run specific test method
        suite.addTest(test_class(test_name))
    elif test_class:
        # Run all tests in the class
        suite.addTests(loader.loadTestsFromTestCase(test_class))
    else:
        # Run all tests
        suite.addTests(loader.loadTestsFromModule(sys.modules[__name__]))

    runner = unittest.TextTestRunner(verbosity=verbosity)
    return runner.run(suite)
run_truth_seeker_tests(test_name=None)

Run TestTruthSeeker tests

Source code in toolboxv2/mods/TruthSeeker/tests.py
384
385
386
def run_truth_seeker_tests(test_name=None):
    """Run TestTruthSeeker tests"""
    return run_test_suite(TestTruthSeeker, test_name)

Run only ArXiv search tests

Source code in toolboxv2/mods/TruthSeeker/tests.py
414
415
416
417
418
419
420
@default_test
def test_arxiv_search():
    """Run only ArXiv search tests"""
    return run_specific_test(
        TestArXivPDFProcessor,
        'test_search_and_process_papers'
    )
test_pdf_download()

Run only PDF download tests

Source code in toolboxv2/mods/TruthSeeker/tests.py
398
399
400
401
402
403
404
@default_test
def test_pdf_download():
    """Run only PDF download tests"""
    return run_specific_test(
        TestRobustPDFDownloader,
        'test_download_pdf_success'
    )
test_truth_seeker()

Run only PDF download tests

Source code in toolboxv2/mods/TruthSeeker/tests.py
406
407
408
409
410
411
412
@default_test
def test_truth_seeker():
    """Run only PDF download tests"""
    return run_specific_test(
        TestTruthSeeker,
        'test_truth_seeker_success'
    )

UltimateTTT

UltimateTTTGameEngine

Source code in toolboxv2/mods/UltimateTTT.py
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
class UltimateTTTGameEngine:  # Renamed for clarity
    def __init__(self, game_state: GameState):
        self.gs = game_state
        self.size = game_state.config.grid_size

    def _check_line_for_win(self, line: list[CellState | BoardWinner],
                            symbol_to_check: CellState | BoardWinner) -> bool:
        if not line or line[0] == CellState.EMPTY or line[0] == BoardWinner.NONE:
            return False
        return all(cell == symbol_to_check for cell in line)

    def _get_board_winner_symbol(self, board: list[list[CellState | BoardWinner]],
                                 symbol_class: type[CellState] | type[BoardWinner]) -> CellState | BoardWinner | None:
        symbols_to_try = [symbol_class.X, symbol_class.O]
        for symbol in symbols_to_try:
            # Rows
            for r in range(self.size):
                if self._check_line_for_win([board[r][c] for c in range(self.size)], symbol): return symbol
            # Columns
            for c in range(self.size):
                if self._check_line_for_win([board[r][c] for r in range(self.size)], symbol): return symbol
            # Diagonals
            if self._check_line_for_win([board[i][i] for i in range(self.size)], symbol): return symbol
            if self._check_line_for_win([board[i][self.size - 1 - i] for i in range(self.size)], symbol): return symbol
        return None  # No winner

    def _is_board_full(self, board: list[list[CellState | BoardWinner]],
                       empty_value: CellState | BoardWinner) -> bool:
        return all(cell != empty_value for row in board for cell in row)

    def _determine_local_board_result(self, global_r: int, global_c: int) -> BoardWinner:
        if self.gs.global_board_winners[global_r][global_c] != BoardWinner.NONE:
            return self.gs.global_board_winners[global_r][global_c]

        local_board_cells = self.gs.local_boards_state[global_r][global_c]
        winner_symbol = self._get_board_winner_symbol(local_board_cells, CellState)
        if winner_symbol:
            return BoardWinner(winner_symbol.value)  # Convert CellState.X to BoardWinner.X
        if self._is_board_full(local_board_cells, CellState.EMPTY):
            return BoardWinner.DRAW
        return BoardWinner.NONE

    def _update_local_winner_and_check_global(self, global_r: int, global_c: int):
        new_local_winner = self._determine_local_board_result(global_r, global_c)
        if new_local_winner != BoardWinner.NONE and self.gs.global_board_winners[global_r][
            global_c] == BoardWinner.NONE:
            self.gs.global_board_winners[global_r][global_c] = new_local_winner
            self._check_for_overall_game_end()

    def _check_for_overall_game_end(self):
        if self.gs.status == GameStatus.FINISHED: return

        winner_board_symbol = self._get_board_winner_symbol(self.gs.global_board_winners, BoardWinner)
        if winner_board_symbol:  # This is BoardWinner.X or BoardWinner.O
            self.gs.overall_winner_symbol = PlayerSymbol(winner_board_symbol.value)  # Convert to PlayerSymbol
            self.gs.status = GameStatus.FINISHED
            return

        if self._is_board_full(self.gs.global_board_winners, BoardWinner.NONE):
            self.gs.is_draw = True
            self.gs.status = GameStatus.FINISHED

    def _determine_next_forced_board(self, last_move_local_r: int, last_move_local_c: int) -> tuple[int, int] | None:
        target_gr, target_gc = last_move_local_r, last_move_local_c

        if self.gs.global_board_winners[target_gr][target_gc] == BoardWinner.NONE and \
            not self._is_local_board_full(self.gs.local_boards_state[target_gr][target_gc], CellState.EMPTY):
            return (target_gr, target_gc)
        return None  # Play anywhere valid

    def _is_local_board_full(self, local_board_cells: list[list[CellState]], cell_type=CellState.EMPTY) -> bool:
        """Checks if a specific local board (passed as a 2D list of CellState) is full."""
        for r in range(self.size):
            for c in range(self.size):
                if local_board_cells[r][c] == cell_type:
                    return False
        return True

    def add_player(self, player_id: str, player_name: str,
                   is_npc: bool = False, npc_difficulty: NPCDifficulty | None = None) -> bool:
        if len(self.gs.players) >= 2:
            self.gs.last_error_message = "Game is already full (2 players max)."
            return False

        # Reconnect logic for existing player (human or NPC if that makes sense)
        existing_player = self.gs.get_player_info(player_id)
        if existing_player:
            if not existing_player.is_connected:
                existing_player.is_connected = True
                # If NPC "reconnects", ensure its properties are correct (though unlikely scenario for NPC)
                if is_npc:
                    existing_player.is_npc = True
                    existing_player.npc_difficulty = npc_difficulty
                    existing_player.name = player_name  # Update name if it changed for NPC

                self.gs.last_error_message = None
                self.gs.updated_at = datetime.now(UTC)

                if len(self.gs.players) == 2 and all(p.is_connected for p in self.gs.players) and \
                    self.gs.status == GameStatus.WAITING_FOR_OPPONENT:  # Should not be waiting if NPC is P2
                    self.gs.status = GameStatus.IN_PROGRESS
                    player_x_info = next(p for p in self.gs.players if p.symbol == PlayerSymbol.X)
                    self.gs.current_player_id = player_x_info.id
                    self.gs.waiting_since = None
                return True
            else:  # Player ID exists and is already connected
                self.gs.last_error_message = f"Player with ID {player_id} is already in the game and connected."
                return False

        # Adding a new player
        symbol = PlayerSymbol.X if not self.gs.players else PlayerSymbol.O

        # Construct PlayerInfo with NPC details if applicable
        player_info_data = {
            "id": player_id,
            "symbol": symbol,
            "name": player_name,
            "is_connected": True,  # NPCs are always "connected"
            "is_npc": is_npc
        }
        if is_npc and npc_difficulty:
            player_info_data["npc_difficulty"] = npc_difficulty

        new_player = PlayerInfo(**player_info_data)
        self.gs.players.append(new_player)
        self.gs.last_error_message = None

        if len(self.gs.players) == 1:  # First player added
            if self.gs.mode == GameMode.ONLINE:
                self.gs.status = GameStatus.WAITING_FOR_OPPONENT
                self.gs.current_player_id = player_id
                self.gs.waiting_since = datetime.now(UTC)
            # For local mode with P1, we wait for P2 (human or NPC) to be added
            # No status change yet, current_player_id not set until P2 joins

        elif len(self.gs.players) == 2:  # Both players now present
            self.gs.status = GameStatus.IN_PROGRESS
            player_x_info = next(p for p in self.gs.players if p.symbol == PlayerSymbol.X)
            self.gs.current_player_id = player_x_info.id  # X always starts
            self.gs.next_forced_global_board = None
            self.gs.waiting_since = None

            # If the second player added is an NPC and it's their turn (e.g. P1 is human, P2 is NPC, P1 made a move)
            # This specific logic is more for when make_move hands over to an NPC.
            # Here, we just set up the game. X (P1) will make the first move.

        self.gs.updated_at = datetime.now(UTC)
        return True

    def make_move(self, move: Move) -> bool:
        self.gs.last_error_message = None

        if self.gs.status != GameStatus.IN_PROGRESS:
            self.gs.last_error_message = "Game is not in progress."
            return False

        player_info = self.gs.get_player_info(move.player_id)
        if not player_info or move.player_id != self.gs.current_player_id:
            self.gs.last_error_message = "Not your turn or invalid player."
            return False

        s = self.size
        if not (0 <= move.global_row < s and 0 <= move.global_col < s and \
                0 <= move.local_row < s and 0 <= move.local_col < s):
            self.gs.last_error_message = f"Coordinates out of bounds for {s}x{s} grid."
            return False

        gr, gc, lr, lc = move.global_row, move.global_col, move.local_row, move.local_col

        if self.gs.next_forced_global_board and (gr, gc) != self.gs.next_forced_global_board:
            self.gs.last_error_message = f"Must play in global board {self.gs.next_forced_global_board}."
            return False

        if self.gs.global_board_winners[gr][gc] != BoardWinner.NONE:
            self.gs.last_error_message = f"Local board ({gr},{gc}) is already decided."
            return False
        if self.gs.local_boards_state[gr][gc][lr][lc] != CellState.EMPTY:
            self.gs.last_error_message = f"Cell ({gr},{gc})-({lr},{lc}) is already empty."  # Should be 'not empty' or 'occupied'
            # Correction:
            self.gs.last_error_message = f"Cell ({gr},{gc})-({lr},{lc}) is already occupied."
            return False

        self.gs.local_boards_state[gr][gc][lr][lc] = CellState(player_info.symbol.value)
        self.gs.moves_history.append(move)

        self._update_local_winner_and_check_global(gr, gc)

        if self.gs.status == GameStatus.FINISHED:
            self.gs.next_forced_global_board = None
        else:
            opponent_info = self.gs.get_opponent_info(self.gs.current_player_id)
            self.gs.current_player_id = opponent_info.id
            self.gs.next_forced_global_board = self._determine_next_forced_board(lr, lc)

            if self.gs.next_forced_global_board is None:
                is_any_move_possible = any(
                    self.gs.global_board_winners[r_idx][c_idx] == BoardWinner.NONE and \
                    not self._is_local_board_full(self.gs.local_boards_state[r_idx][c_idx], CellState.EMPTY)
                    for r_idx in range(s) for c_idx in range(s)
                )
                if not is_any_move_possible:
                    self._check_for_overall_game_end()
                    if self.gs.status != GameStatus.FINISHED:
                        self.gs.is_draw = True
                        self.gs.status = GameStatus.FINISHED

        self.gs.updated_at = datetime.now(UTC)
        self.gs.last_made_move_coords = (move.global_row, move.global_col, move.local_row, move.local_col)

        return True

    def handle_player_disconnect(self, player_id: str):
        player = self.gs.get_player_info(player_id)
        app = get_app(GAME_NAME)  # Hol dir die App-Instanz
        if player:
            if not player.is_connected:  # Already marked as disconnected
                app.logger.info(f"Player {player_id} was already marked as disconnected from game {self.gs.game_id}.")
                return

            player.is_connected = False
            self.gs.updated_at = datetime.now(UTC)
            app.logger.info(f"Player {player_id} disconnected from game {self.gs.game_id}. Name: {player.name}")

            if self.gs.mode == GameMode.ONLINE:
                if self.gs.status == GameStatus.IN_PROGRESS:
                    opponent = self.gs.get_opponent_info(player_id)
                    if opponent and opponent.is_connected:
                        self.gs.status = GameStatus.ABORTED  # Use ABORTED as "paused"
                        self.gs.player_who_paused = player_id  # Store who disconnected
                        # This message is for the game state, will be seen by the other player via SSE
                        self.gs.last_error_message = f"Player {player.name} disconnected. Waiting for them to rejoin."
                        app.logger.info(
                            f"Game {self.gs.game_id} PAUSED, waiting for {player.name} ({player_id}) to reconnect.")
                    else:
                        # Opponent also disconnected or was already gone
                        self.gs.status = GameStatus.ABORTED
                        self.gs.last_error_message = "Both players disconnected. Game aborted."
                        self.gs.player_who_paused = None  # No specific player to wait for
                        app.logger.info(
                            f"Game {self.gs.game_id} ABORTED, both players (or last active player) disconnected.")
                elif self.gs.status == GameStatus.WAITING_FOR_OPPONENT:
                    # If the creator (P1) disconnects while waiting for P2
                    if len(self.gs.players) == 1 and self.gs.players[0].id == player_id:
                        self.gs.status = GameStatus.ABORTED
                        self.gs.last_error_message = "Game creator disconnected before opponent joined. Game aborted."
                        self.gs.player_who_paused = None
                        app.logger.info(
                            f"Game {self.gs.game_id} ABORTED, creator {player.name} ({player_id}) disconnected while WAITING_FOR_OPPONENT.")
                elif self.gs.status == GameStatus.ABORTED and self.gs.player_who_paused:
                    # Game was already paused (e.g. P1 disconnected), and now P2 (the waiting one) disconnects
                    if self.gs.player_who_paused != player_id:  # Ensure it's the other player
                        self.gs.last_error_message = "Other player also disconnected during pause. Game aborted."
                        self.gs.player_who_paused = None  # No one specific to wait for now
                        app.logger.info(
                            f"Game {self.gs.game_id} ABORTED, waiting player {player.name} ({player_id}) disconnected.")

    def handle_player_reconnect(self, player_id: str) -> bool:
        player = self.gs.get_player_info(player_id)
        app = get_app(GAME_NAME)
        if not player:
            app.logger.warning(f"Reconnect attempt for unknown player {player_id} in game {self.gs.game_id}.")
            return False

        if player.is_connected:
            app.logger.info(
                f"Player {player.name} ({player_id}) attempted reconnect but was already marked as connected to game {self.gs.game_id}.")
            if self.gs.status == GameStatus.ABORTED and self.gs.player_who_paused == player_id:
                opponent = self.gs.get_opponent_info(player_id)
                if opponent and opponent.is_connected:
                    self.gs.status = GameStatus.IN_PROGRESS
                    self.gs.last_error_message = f"Connection for {player.name} re-established. Game resumed."
                    self.gs.player_who_paused = None
                    self.gs.updated_at = datetime.now(UTC)
                    app.logger.info(
                        f"Game {self.gs.game_id} resumed as already-connected pauser {player.name} re-interacted.")
                else:
                    self.gs.last_error_message = f"Welcome back, {player.name}! Your opponent is still not connected."
            return True

        player.is_connected = True
        self.gs.updated_at = datetime.now(UTC)
        app.logger.info(
            f"Player {player.name} ({player_id}) reconnected to game {self.gs.game_id}. Previous status: {self.gs.status}, Paused by: {self.gs.player_who_paused}")

        if self.gs.status == GameStatus.ABORTED:
            if self.gs.player_who_paused == player_id:  # The player who caused the pause has reconnected
                opponent = self.gs.get_opponent_info(player_id)
                if opponent and opponent.is_connected:
                    self.gs.status = GameStatus.IN_PROGRESS
                    self.gs.last_error_message = f"Player {player.name} reconnected. Game resumed!"
                    self.gs.player_who_paused = None
                    app.logger.info(
                        f"Game {self.gs.game_id} RESUMED. Pauser {player.name} reconnected, opponent {opponent.name} is present.")
                else:  # Pauser reconnected, opponent (still) gone or never joined (if P1 disconnected from WAITING)
                    if not opponent and len(
                        self.gs.players) == 1:  # P1 reconnected to a game they created but no P2 yet
                        self.gs.status = GameStatus.WAITING_FOR_OPPONENT
                        self.gs.player_who_paused = None
                        self.gs.current_player_id = player_id
                        self.gs.last_error_message = f"Creator {player.name} reconnected. Waiting for opponent."
                        self.gs.waiting_since = datetime.now(UTC)  # Reset waiting timer
                    elif opponent:  # Opponent was there but is now disconnected
                        self.gs.player_who_paused = opponent.id  # Now waiting for the other person
                        self.gs.last_error_message = f"Welcome back, {player.name}! Your opponent ({opponent.name}) is not connected. Game remains paused."
                        app.logger.info(
                            f"Game {self.gs.game_id} still PAUSED. {player.name} reconnected, but opponent {opponent.name} is NOT. Waiting for {opponent.name}.")
                    else:  # Should be rare: 2 players in list, but opponent object not found for P1
                        self.gs.last_error_message = f"Welcome back, {player.name}! Opponent details unclear. Game remains paused."


            elif self.gs.player_who_paused and self.gs.player_who_paused != player_id:
                # The *other* player reconnected, while game was paused for initial pauser.
                initial_pauser_info = self.gs.get_player_info(self.gs.player_who_paused)
                if initial_pauser_info and initial_pauser_info.is_connected:  # This implies both are now connected.
                    self.gs.status = GameStatus.IN_PROGRESS
                    self.gs.last_error_message = "Both players are now connected. Game resumed!"
                    self.gs.player_who_paused = None
                    app.logger.info(
                        f"Game {self.gs.game_id} RESUMED. Waiting player {player.name} reconnected, initial pauser {initial_pauser_info.name} also present.")
                else:
                    self.gs.last_error_message = f"Welcome back, {player.name}! Still waiting for {initial_pauser_info.name if initial_pauser_info else 'the other player'} to reconnect."
                    app.logger.info(
                        f"Game {self.gs.game_id} still PAUSED. Player {player.name} reconnected, but still waiting for original pauser {self.gs.player_who_paused}.")

            else:  # game is ABORTED but no specific player_who_paused (hard abort by timeout or both disconnected)
                if len(self.gs.players) == 2:  # Was a two-player game
                    opponent = self.gs.get_opponent_info(player_id)
                    if opponent:
                        # Revive the game to a paused state, waiting for the other player
                        self.gs.player_who_paused = opponent.id
                        self.gs.status = GameStatus.ABORTED  # Remains aborted, but now specifically for opponent
                        self.gs.last_error_message = f"Welcome back, {player.name}! Game was fully aborted. Now waiting for {opponent.name} to rejoin."
                        app.logger.info(
                            f"Game {self.gs.game_id} REVIVED from HARD ABORT by {player.name}. Now paused, waiting for {opponent.name} ({opponent.id}).")
                    else:  # Should not happen if two players were in game and player_id is one of them
                        self.gs.last_error_message = f"Player {player.name} reconnected, but game state is inconsistent (opponent not found)."
                        app.logger.warning(
                            f"Game {self.gs.game_id} HARD ABORT revival by {player.name} failed, opponent info missing.")
                elif len(self.gs.players) == 1 and self.gs.players[0].id == player_id:
                    # P1 created, P1 disconnected, game WAITING_FOR_OPPONENT timed out & hard aborted. P1 tries to rejoin.
                    self.gs.status = GameStatus.WAITING_FOR_OPPONENT
                    self.gs.player_who_paused = None
                    self.gs.current_player_id = player_id
                    self.gs.last_error_message = f"Creator {player.name} reconnected. Waiting for opponent."
                    self.gs.waiting_since = datetime.now(UTC)  # Reset waiting timer
                    app.logger.info(
                        f"Game {self.gs.game_id} (previously hard aborted while waiting) revived by creator {player.name}. Now WAITING_FOR_OPPONENT.")
                else:
                    self.gs.last_error_message = f"Player {player.name} reconnected, but the game was aborted and cannot be revived in its current player configuration."
                    app.logger.info(
                        f"Game {self.gs.game_id} HARD ABORTED. Player {player.name} reconnected, but game cannot resume in current configuration.")


        elif self.gs.status == GameStatus.IN_PROGRESS:
            opponent = self.gs.get_opponent_info(player_id)
            if not opponent or not opponent.is_connected:
                self.gs.status = GameStatus.ABORTED
                self.gs.player_who_paused = opponent.id if opponent else None
                self.gs.last_error_message = f"Welcome back, {player.name}! Your opponent disconnected while you were away. Waiting for them."
                app.logger.info(
                    f"Game {self.gs.game_id} transitions to PAUSED. {player.name} reconnected to IN_PROGRESS, but opponent {opponent.id if opponent else 'N/A'} is gone.")
            else:
                self.gs.last_error_message = f"Player {player.name} re-established connection during active game."
                app.logger.info(
                    f"Player {player.name} ({player_id}) re-established connection to IN_PROGRESS game {self.gs.game_id}.")

        elif self.gs.status == GameStatus.WAITING_FOR_OPPONENT:
            if len(self.gs.players) == 1 and self.gs.players[0].id == player_id:
                self.gs.last_error_message = f"Creator {player.name} reconnected. Still waiting for opponent."
                self.gs.current_player_id = player_id
                self.gs.waiting_since = datetime.now(UTC)  # Reset waiting timer
                app.logger.info(
                    f"Creator {player.name} ({player_id}) reconnected to WAITING_FOR_OPPONENT game {self.gs.game_id}.")
            else:
                app.logger.warning(
                    f"Non-creator {player.name} or unexpected player count for reconnect to WAITING_FOR_OPPONENT game {self.gs.game_id}.")

        return True

WebSocketManager

Tools

Bases: MainTool

Production-ready WebSocketManager Tool.

Source code in toolboxv2/mods/WebSocketManager.py
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
class Tools(MainTool):
    """Production-ready WebSocketManager Tool."""

    def __init__(self, app=None):
        self.version = "2.0.0"
        self.name = "WebSocketManager"
        self.color = "BLUE"

        if app is None:
            app = get_app()
        self.logger = app.logger if app else logging.getLogger(self.name)

        # Core components
        self.server: Optional[WebSocketServer] = None
        self.clients: Dict[str, WebSocketClient] = {}
        self.pools: Dict[str, WebSocketPool] = {}

        # Tools interface
        self.tools = {
            "all": [
                ["version", "Show version"],
                ["create_server", "Create WebSocket server"],
                ["create_client", "Create WebSocket client"],
                ["create_pool", "Create connection pool"],
                ["list_pools", "List all pools"],
                ["get_stats", "Get connection statistics"],
                ["health_check", "Perform health check"]
            ],
            "name": self.name,
            "version": self.show_version,
            #"create_server": self.create_server,
            "create_client": self.create_client,
            "create_pool": self.create_pool,
            "list_pools": self.list_pools,
            "get_stats": self.get_statistics,
            "health_check": self.health_check
        }

        MainTool.__init__(self, load=self.on_start, v=self.version,
                          tool=self.tools, name=self.name,
                          logs=self.logger, color=self.color,
                          on_exit=self.on_exit)

    def on_start(self):
        """Initialize the WebSocketManager."""
        self.logger.info("🚀 WebSocketManager started")

    async def on_exit(self):
        """Cleanup on exit."""
        self.logger.info("🔄 Shutting down WebSocketManager")

        # Stop server
        if self.server:
            await self.server.stop()

        # Disconnect all clients
        for client in self.clients.values():
            await client.disconnect()

        self.logger.info("✅ WebSocketManager shutdown complete")

    def show_version(self):
        """Show current version."""
        return self.version

    async def create_server(self, host: str = "localhost", port: int = 8765,
                            non_blocking: bool = False) -> WebSocketServer:
        """Create and start a WebSocket server."""
        if non_blocking is None:
            return
        if 'test' in host:
            return
        if self.server is None:
            self.server = WebSocketServer(host, port)
            await self.server.start(non_blocking)
        return self.server

    def create_client(self, client_id: str) -> WebSocketClient:
        """Create a WebSocket client."""
        if client_id not in self.clients:
            self.clients[client_id] = WebSocketClient(client_id, self.logger)
        return self.clients[client_id]

    def create_pool(self, pool_id: str) -> WebSocketPool:
        """Create a standalone connection pool."""
        if pool_id not in self.pools:
            self.pools[pool_id] = WebSocketPool(pool_id)
        return self.pools[pool_id]

    def list_pools(self) -> Dict[str, Dict[str, Any]]:
        """List all connection pools with stats."""
        pools_info = {}

        # Server pools
        if self.server:
            for pool_id, pool in self.server.pools.items():
                pools_info[f"server.{pool_id}"] = {
                    "type": "server_pool",
                    "connections": pool.get_connection_count(),
                    "connection_ids": pool.get_connection_ids()
                }

        # Standalone pools
        for pool_id, pool in self.pools.items():
            pools_info[pool_id] = {
                "type": "standalone_pool",
                "connections": pool.get_connection_count(),
                "connection_ids": pool.get_connection_ids()
            }

        return pools_info

    def get_statistics(self) -> Dict[str, Any]:
        """Get comprehensive statistics."""
        stats = {
            "server": {
                "running": self.server is not None,
                "pools": len(self.server.pools) if self.server else 0,
                "total_connections": sum(
                    pool.get_connection_count()
                    for pool in (self.server.pools.values() if self.server else [])
                )
            },
            "clients": {
                "total": len(self.clients),
                "connected": sum(
                    1 for client in self.clients.values()
                    if client.state == ConnectionState.CONNECTED
                ),
                "states": {
                    state.value: sum(
                        1 for client in self.clients.values()
                        if client.state == state
                    ) for state in ConnectionState
                }
            },
            "pools": {
                "standalone": len(self.pools),
                "total_connections": sum(
                    pool.get_connection_count()
                    for pool in self.pools.values()
                )
            }
        }
        return stats

    async def health_check(self) -> Dict[str, Any]:
        """Perform comprehensive health check."""
        health = {
            "overall": "healthy",
            "server": "not_running" if not self.server else "running",
            "clients": {},
            "issues": []
        }

        # Check clients
        for client_id, client in self.clients.items():
            if client.state == ConnectionState.CONNECTED:
                # Perform actual health check if possible
                try:
                    if client.ws and not client.ws.closed:
                        health["clients"][client_id] = "healthy"
                    else:
                        health["clients"][client_id] = "unhealthy"
                        health["issues"].append(f"Client {client_id} connection closed")
                except Exception as e:
                    health["clients"][client_id] = "error"
                    health["issues"].append(f"Client {client_id}: {str(e)}")
            else:
                health["clients"][client_id] = client.state.value

        if health["issues"]:
            health["overall"] = "degraded"

        return health

    # Utility methods for easy access
    def get_server_pool(self, pool_id: str) -> Optional[WebSocketPool]:
        """Get a server pool by ID."""
        return self.server.get_pool(pool_id) if self.server else None

    def get_client(self, client_id: str) -> Optional[WebSocketClient]:
        """Get a client by ID."""
        return self.clients.get(client_id)

    async def broadcast_to_pool(self, pool_id: str, event: str, data: Dict[str, Any]) -> int:
        """Broadcast message to all connections in a pool."""
        message = WebSocketMessage(event=event, data=data).to_json()

        # Try server pool first
        if self.server:
            pool = self.server.get_pool(pool_id)
            if pool:
                return await pool.broadcast(message)

        # Try standalone pool
        pool = self.pools.get(pool_id)
        if pool:
            return await pool.broadcast(message)

        return 0
broadcast_to_pool(pool_id, event, data) async

Broadcast message to all connections in a pool.

Source code in toolboxv2/mods/WebSocketManager.py
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
async def broadcast_to_pool(self, pool_id: str, event: str, data: Dict[str, Any]) -> int:
    """Broadcast message to all connections in a pool."""
    message = WebSocketMessage(event=event, data=data).to_json()

    # Try server pool first
    if self.server:
        pool = self.server.get_pool(pool_id)
        if pool:
            return await pool.broadcast(message)

    # Try standalone pool
    pool = self.pools.get(pool_id)
    if pool:
        return await pool.broadcast(message)

    return 0
create_client(client_id)

Create a WebSocket client.

Source code in toolboxv2/mods/WebSocketManager.py
511
512
513
514
515
def create_client(self, client_id: str) -> WebSocketClient:
    """Create a WebSocket client."""
    if client_id not in self.clients:
        self.clients[client_id] = WebSocketClient(client_id, self.logger)
    return self.clients[client_id]
create_pool(pool_id)

Create a standalone connection pool.

Source code in toolboxv2/mods/WebSocketManager.py
517
518
519
520
521
def create_pool(self, pool_id: str) -> WebSocketPool:
    """Create a standalone connection pool."""
    if pool_id not in self.pools:
        self.pools[pool_id] = WebSocketPool(pool_id)
    return self.pools[pool_id]
create_server(host='localhost', port=8765, non_blocking=False) async

Create and start a WebSocket server.

Source code in toolboxv2/mods/WebSocketManager.py
499
500
501
502
503
504
505
506
507
508
509
async def create_server(self, host: str = "localhost", port: int = 8765,
                        non_blocking: bool = False) -> WebSocketServer:
    """Create and start a WebSocket server."""
    if non_blocking is None:
        return
    if 'test' in host:
        return
    if self.server is None:
        self.server = WebSocketServer(host, port)
        await self.server.start(non_blocking)
    return self.server
get_client(client_id)

Get a client by ID.

Source code in toolboxv2/mods/WebSocketManager.py
615
616
617
def get_client(self, client_id: str) -> Optional[WebSocketClient]:
    """Get a client by ID."""
    return self.clients.get(client_id)
get_server_pool(pool_id)

Get a server pool by ID.

Source code in toolboxv2/mods/WebSocketManager.py
611
612
613
def get_server_pool(self, pool_id: str) -> Optional[WebSocketPool]:
    """Get a server pool by ID."""
    return self.server.get_pool(pool_id) if self.server else None
get_statistics()

Get comprehensive statistics.

Source code in toolboxv2/mods/WebSocketManager.py
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
def get_statistics(self) -> Dict[str, Any]:
    """Get comprehensive statistics."""
    stats = {
        "server": {
            "running": self.server is not None,
            "pools": len(self.server.pools) if self.server else 0,
            "total_connections": sum(
                pool.get_connection_count()
                for pool in (self.server.pools.values() if self.server else [])
            )
        },
        "clients": {
            "total": len(self.clients),
            "connected": sum(
                1 for client in self.clients.values()
                if client.state == ConnectionState.CONNECTED
            ),
            "states": {
                state.value: sum(
                    1 for client in self.clients.values()
                    if client.state == state
                ) for state in ConnectionState
            }
        },
        "pools": {
            "standalone": len(self.pools),
            "total_connections": sum(
                pool.get_connection_count()
                for pool in self.pools.values()
            )
        }
    }
    return stats
health_check() async

Perform comprehensive health check.

Source code in toolboxv2/mods/WebSocketManager.py
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
async def health_check(self) -> Dict[str, Any]:
    """Perform comprehensive health check."""
    health = {
        "overall": "healthy",
        "server": "not_running" if not self.server else "running",
        "clients": {},
        "issues": []
    }

    # Check clients
    for client_id, client in self.clients.items():
        if client.state == ConnectionState.CONNECTED:
            # Perform actual health check if possible
            try:
                if client.ws and not client.ws.closed:
                    health["clients"][client_id] = "healthy"
                else:
                    health["clients"][client_id] = "unhealthy"
                    health["issues"].append(f"Client {client_id} connection closed")
            except Exception as e:
                health["clients"][client_id] = "error"
                health["issues"].append(f"Client {client_id}: {str(e)}")
        else:
            health["clients"][client_id] = client.state.value

    if health["issues"]:
        health["overall"] = "degraded"

    return health
list_pools()

List all connection pools with stats.

Source code in toolboxv2/mods/WebSocketManager.py
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
def list_pools(self) -> Dict[str, Dict[str, Any]]:
    """List all connection pools with stats."""
    pools_info = {}

    # Server pools
    if self.server:
        for pool_id, pool in self.server.pools.items():
            pools_info[f"server.{pool_id}"] = {
                "type": "server_pool",
                "connections": pool.get_connection_count(),
                "connection_ids": pool.get_connection_ids()
            }

    # Standalone pools
    for pool_id, pool in self.pools.items():
        pools_info[pool_id] = {
            "type": "standalone_pool",
            "connections": pool.get_connection_count(),
            "connection_ids": pool.get_connection_ids()
        }

    return pools_info
on_exit() async

Cleanup on exit.

Source code in toolboxv2/mods/WebSocketManager.py
481
482
483
484
485
486
487
488
489
490
491
492
493
async def on_exit(self):
    """Cleanup on exit."""
    self.logger.info("🔄 Shutting down WebSocketManager")

    # Stop server
    if self.server:
        await self.server.stop()

    # Disconnect all clients
    for client in self.clients.values():
        await client.disconnect()

    self.logger.info("✅ WebSocketManager shutdown complete")
on_start()

Initialize the WebSocketManager.

Source code in toolboxv2/mods/WebSocketManager.py
477
478
479
def on_start(self):
    """Initialize the WebSocketManager."""
    self.logger.info("🚀 WebSocketManager started")
show_version()

Show current version.

Source code in toolboxv2/mods/WebSocketManager.py
495
496
497
def show_version(self):
    """Show current version."""
    return self.version

WebSocketClient

Robust WebSocket client with automatic reconnection.

Source code in toolboxv2/mods/WebSocketManager.py
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
class WebSocketClient:
    """Robust WebSocket client with automatic reconnection."""

    def __init__(self, client_id: str, logger: Optional[logging.Logger] = None):
        self.client_id = client_id
        self.logger = logger or logging.getLogger(f"WSClient.{client_id}")

        # Connection management
        self.ws: Optional[Any] = None
        self.server_url: Optional[str] = None
        self.state = ConnectionState.DISCONNECTED

        # Tasks and control
        self.should_reconnect = True
        self.reconnect_attempts = 0
        self.max_reconnect_attempts = 10
        self.connection_task: Optional[asyncio.Task] = None
        self.ping_task: Optional[asyncio.Task] = None

        # Message handling
        self.message_handlers: Dict[str, Callable] = {}
        self.message_queue = asyncio.Queue()

    async def connect(self, server_url: str, timeout: float = 30.0) -> bool:
        """Connect to WebSocket server."""
        if self.state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]:
            return True

        self.server_url = server_url
        self.state = ConnectionState.CONNECTING
        self.should_reconnect = True

        try:
            self.logger.info(f"Connecting to {server_url}")
            self.ws = await asyncio.wait_for(ws_connect(server_url), timeout=timeout)

            self.state = ConnectionState.CONNECTED
            self.reconnect_attempts = 0

            # Start background tasks
            self.connection_task = asyncio.create_task(self._listen_loop())
            self.ping_task = asyncio.create_task(self._ping_loop())

            self.logger.info("✅ Connected successfully")
            return True

        except Exception as e:
            self.logger.error(f"❌ Connection failed: {e}")
            self.state = ConnectionState.DISCONNECTED
            return False

    async def disconnect(self) -> None:
        """Gracefully disconnect."""
        self.should_reconnect = False
        self.state = ConnectionState.CLOSED

        # Cancel tasks
        for task in [self.connection_task, self.ping_task]:
            if task and not task.done():
                task.cancel()

        # Close connection
        if self.ws:
            try:
                await self.ws.close()
            except Exception:
                pass
            self.ws = None

        self.logger.info("✅ Disconnected")

    def register_handler(self, event: str, handler: Callable[[WebSocketMessage], Awaitable[None]]) -> None:
        """Register a message handler for specific events."""
        self.message_handlers[event] = handler
        self.logger.info(f"Registered handler for event: {event}")

    async def send_message(self, event: str, data: Dict[str, Any]) -> bool:
        """Send a message to the server."""
        if self.state != ConnectionState.CONNECTED or not self.ws:
            self.logger.warning("Cannot send message: not connected")
            return False

        try:
            message = WebSocketMessage(event=event, data=data)
            await self.ws.send(message.to_json())
            return True
        except Exception as e:
            self.logger.error(f"Failed to send message: {e}")
            await self._trigger_reconnect()
            return False

    async def _listen_loop(self) -> None:
        """Main message listening loop."""
        while self.should_reconnect and self.ws:
            try:
                # Kürzere Timeouts für bessere Responsivität
                message_raw = await asyncio.wait_for(self.ws.recv(), timeout=1.0)

                # Handle message in background task to prevent blocking
                asyncio.create_task(self._handle_message(message_raw))

            except asyncio.TimeoutError:
                # Check connection health during timeout
                if self.ws and self.ws.closed:
                    self.logger.warning("Connection closed during timeout")
                    break
                continue
            except ConnectionClosed:
                self.logger.warning("Connection closed by server")
                break
            except Exception as e:
                self.logger.error(f"Listen loop error: {e}")
                break

        if self.should_reconnect:
            await self._trigger_reconnect()

    async def _handle_message(self, message_raw: str) -> None:
        """Handle incoming messages."""
        try:
            message = WebSocketMessage.from_json(message_raw)

            if message.event in self.message_handlers:
                await self.message_handlers[message.event](message)
            else:
                self.logger.debug(f"No handler for event: {message.event}")

        except Exception as e:
            self.logger.error(f"Message handling error: {e}")

    async def _ping_loop(self) -> None:
        """Periodic ping to maintain connection."""
        while self.should_reconnect and self.state == ConnectionState.CONNECTED:
            try:
                await asyncio.sleep(20)  # Ping every 20 seconds

                if self.ws and not self.ws.closed:
                    pong_waiter = await self.ws.ping()
                    await asyncio.wait_for(pong_waiter, timeout=10.0)
                    self.logger.debug("📡 Ping successful")
                else:
                    break

            except asyncio.TimeoutError:
                self.logger.error("Ping timeout - connection may be dead")
                break
            except Exception as e:
                self.logger.error(f"Ping failed: {e}")
                break

        if self.should_reconnect:
            await self._trigger_reconnect()

    async def _trigger_reconnect(self) -> None:
        """Trigger reconnection with exponential backoff."""
        if self.state == ConnectionState.RECONNECTING:
            return

        self.state = ConnectionState.RECONNECTING
        self.logger.info("🔄 Starting reconnection...")

        while (self.should_reconnect and
               self.reconnect_attempts < self.max_reconnect_attempts):

            self.reconnect_attempts += 1
            delay = min(2 ** self.reconnect_attempts, 60)  # Max 60s delay

            self.logger.info(f"Reconnect attempt {self.reconnect_attempts} in {delay}s")
            await asyncio.sleep(delay)

            try:
                if await self.connect(self.server_url):
                    return
            except Exception as e:
                self.logger.error(f"Reconnect attempt failed: {e}")

        self.logger.error("❌ Max reconnection attempts reached")
        self.should_reconnect = False
        self.state = ConnectionState.DISCONNECTED
connect(server_url, timeout=30.0) async

Connect to WebSocket server.

Source code in toolboxv2/mods/WebSocketManager.py
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
async def connect(self, server_url: str, timeout: float = 30.0) -> bool:
    """Connect to WebSocket server."""
    if self.state in [ConnectionState.CONNECTED, ConnectionState.CONNECTING]:
        return True

    self.server_url = server_url
    self.state = ConnectionState.CONNECTING
    self.should_reconnect = True

    try:
        self.logger.info(f"Connecting to {server_url}")
        self.ws = await asyncio.wait_for(ws_connect(server_url), timeout=timeout)

        self.state = ConnectionState.CONNECTED
        self.reconnect_attempts = 0

        # Start background tasks
        self.connection_task = asyncio.create_task(self._listen_loop())
        self.ping_task = asyncio.create_task(self._ping_loop())

        self.logger.info("✅ Connected successfully")
        return True

    except Exception as e:
        self.logger.error(f"❌ Connection failed: {e}")
        self.state = ConnectionState.DISCONNECTED
        return False
disconnect() async

Gracefully disconnect.

Source code in toolboxv2/mods/WebSocketManager.py
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
async def disconnect(self) -> None:
    """Gracefully disconnect."""
    self.should_reconnect = False
    self.state = ConnectionState.CLOSED

    # Cancel tasks
    for task in [self.connection_task, self.ping_task]:
        if task and not task.done():
            task.cancel()

    # Close connection
    if self.ws:
        try:
            await self.ws.close()
        except Exception:
            pass
        self.ws = None

    self.logger.info("✅ Disconnected")
register_handler(event, handler)

Register a message handler for specific events.

Source code in toolboxv2/mods/WebSocketManager.py
244
245
246
247
def register_handler(self, event: str, handler: Callable[[WebSocketMessage], Awaitable[None]]) -> None:
    """Register a message handler for specific events."""
    self.message_handlers[event] = handler
    self.logger.info(f"Registered handler for event: {event}")
send_message(event, data) async

Send a message to the server.

Source code in toolboxv2/mods/WebSocketManager.py
249
250
251
252
253
254
255
256
257
258
259
260
261
262
async def send_message(self, event: str, data: Dict[str, Any]) -> bool:
    """Send a message to the server."""
    if self.state != ConnectionState.CONNECTED or not self.ws:
        self.logger.warning("Cannot send message: not connected")
        return False

    try:
        message = WebSocketMessage(event=event, data=data)
        await self.ws.send(message.to_json())
        return True
    except Exception as e:
        self.logger.error(f"Failed to send message: {e}")
        await self._trigger_reconnect()
        return False

WebSocketPool

Manages a pool of WebSocket connections with actions and message routing.

Source code in toolboxv2/mods/WebSocketManager.py
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
class WebSocketPool:
    """Manages a pool of WebSocket connections with actions and message routing."""

    def __init__(self, pool_id: str):
        self.pool_id = pool_id
        self.connections: Dict[str, Any] = {}
        self.actions: Dict[str, Callable] = {}
        self.global_actions: Dict[str, Callable] = {}
        self.metadata: Dict[str, Any] = {}
        self.logger = logging.getLogger(f"WSPool.{pool_id}")

    async def add_connection(self, connection_id: str, websocket: Any) -> None:
        """Add a WebSocket connection to the pool."""
        self.connections[connection_id] = websocket
        self.logger.info(f"Added connection {connection_id} (total: {len(self.connections)})")

    async def remove_connection(self, connection_id: str) -> None:
        """Remove a WebSocket connection from the pool."""
        if connection_id in self.connections:
            del self.connections[connection_id]
            self.logger.info(f"Removed connection {connection_id} (remaining: {len(self.connections)})")

    def register_action(self, action_name: str, handler: Callable,
                        connection_ids: Optional[List[str]] = None) -> None:
        """Register an action handler for specific connections or globally."""
        if connection_ids is None:
            self.global_actions[action_name] = handler
            self.logger.info(f"Registered global action: {action_name}")
        else:
            for conn_id in connection_ids:
                if conn_id not in self.actions:
                    self.actions[conn_id] = {}
                self.actions[conn_id][action_name] = handler
            self.logger.info(f"Registered action {action_name} for connections: {connection_ids}")

    async def handle_message(self, connection_id: str, message: str) -> None:
        """Route incoming messages to appropriate handlers."""
        try:
            ws_message = WebSocketMessage.from_json(message)
            action = ws_message.event

            # Handle ping/pong
            if action == 'ping':
                pong_message = WebSocketMessage(event='pong', data={})
                await self.send_to_connection(connection_id, pong_message.to_json())
                return

            # Try global actions first
            if action in self.global_actions:
                # Run in executor to prevent blocking
                loop = asyncio.get_event_loop()
                await loop.run_in_executor(
                    None,
                    lambda: asyncio.create_task(
                        self.global_actions[action](self.pool_id, connection_id, ws_message)
                    )
                )
            # Then try connection-specific actions
            elif connection_id in self.actions and action in self.actions[connection_id]:
                loop = asyncio.get_event_loop()
                await loop.run_in_executor(
                    None,
                    lambda: asyncio.create_task(
                        self.actions[connection_id][action](self.pool_id, connection_id, ws_message)
                    )
                )
            else:
                self.logger.warning(f"No handler for action '{action}' from {connection_id}")

        except json.JSONDecodeError:
            self.logger.error(f"Invalid JSON from {connection_id}: {message[:100]}")
        except Exception as e:
            self.logger.error(f"Error handling message from {connection_id}: {e}")

    async def broadcast(self, message: str, exclude_connection: Optional[str] = None) -> int:
        """Broadcast message to all connections in the pool."""
        sent_count = 0
        for conn_id, websocket in list(self.connections.items()):
            if conn_id != exclude_connection:
                try:
                    await websocket.send(message)
                    sent_count += 1
                except Exception as e:
                    self.logger.error(f"Failed to send to {conn_id}: {e}")
                    await self.remove_connection(conn_id)
        return sent_count

    async def send_to_connection(self, connection_id: str, message: str) -> bool:
        """Send message to a specific connection."""
        if connection_id in self.connections:
            try:
                await self.connections[connection_id].send(message)
                return True
            except Exception as e:
                self.logger.error(f"Failed to send to {connection_id}: {e}")
                await self.remove_connection(connection_id)
        return False

    def get_connection_ids(self) -> List[str]:
        """Get list of all connection IDs."""
        return list(self.connections.keys())

    def get_connection_count(self) -> int:
        """Get number of active connections."""
        return len(self.connections)

    async def close_all(self) -> None:
        """Close all connections in the pool."""
        for websocket in list(self.connections.values()):
            try:
                await websocket.close()
            except Exception:
                pass
        self.connections.clear()
add_connection(connection_id, websocket) async

Add a WebSocket connection to the pool.

Source code in toolboxv2/mods/WebSocketManager.py
68
69
70
71
async def add_connection(self, connection_id: str, websocket: Any) -> None:
    """Add a WebSocket connection to the pool."""
    self.connections[connection_id] = websocket
    self.logger.info(f"Added connection {connection_id} (total: {len(self.connections)})")
broadcast(message, exclude_connection=None) async

Broadcast message to all connections in the pool.

Source code in toolboxv2/mods/WebSocketManager.py
131
132
133
134
135
136
137
138
139
140
141
142
async def broadcast(self, message: str, exclude_connection: Optional[str] = None) -> int:
    """Broadcast message to all connections in the pool."""
    sent_count = 0
    for conn_id, websocket in list(self.connections.items()):
        if conn_id != exclude_connection:
            try:
                await websocket.send(message)
                sent_count += 1
            except Exception as e:
                self.logger.error(f"Failed to send to {conn_id}: {e}")
                await self.remove_connection(conn_id)
    return sent_count
close_all() async

Close all connections in the pool.

Source code in toolboxv2/mods/WebSocketManager.py
163
164
165
166
167
168
169
170
async def close_all(self) -> None:
    """Close all connections in the pool."""
    for websocket in list(self.connections.values()):
        try:
            await websocket.close()
        except Exception:
            pass
    self.connections.clear()
get_connection_count()

Get number of active connections.

Source code in toolboxv2/mods/WebSocketManager.py
159
160
161
def get_connection_count(self) -> int:
    """Get number of active connections."""
    return len(self.connections)
get_connection_ids()

Get list of all connection IDs.

Source code in toolboxv2/mods/WebSocketManager.py
155
156
157
def get_connection_ids(self) -> List[str]:
    """Get list of all connection IDs."""
    return list(self.connections.keys())
handle_message(connection_id, message) async

Route incoming messages to appropriate handlers.

Source code in toolboxv2/mods/WebSocketManager.py
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
async def handle_message(self, connection_id: str, message: str) -> None:
    """Route incoming messages to appropriate handlers."""
    try:
        ws_message = WebSocketMessage.from_json(message)
        action = ws_message.event

        # Handle ping/pong
        if action == 'ping':
            pong_message = WebSocketMessage(event='pong', data={})
            await self.send_to_connection(connection_id, pong_message.to_json())
            return

        # Try global actions first
        if action in self.global_actions:
            # Run in executor to prevent blocking
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(
                None,
                lambda: asyncio.create_task(
                    self.global_actions[action](self.pool_id, connection_id, ws_message)
                )
            )
        # Then try connection-specific actions
        elif connection_id in self.actions and action in self.actions[connection_id]:
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(
                None,
                lambda: asyncio.create_task(
                    self.actions[connection_id][action](self.pool_id, connection_id, ws_message)
                )
            )
        else:
            self.logger.warning(f"No handler for action '{action}' from {connection_id}")

    except json.JSONDecodeError:
        self.logger.error(f"Invalid JSON from {connection_id}: {message[:100]}")
    except Exception as e:
        self.logger.error(f"Error handling message from {connection_id}: {e}")
register_action(action_name, handler, connection_ids=None)

Register an action handler for specific connections or globally.

Source code in toolboxv2/mods/WebSocketManager.py
79
80
81
82
83
84
85
86
87
88
89
90
def register_action(self, action_name: str, handler: Callable,
                    connection_ids: Optional[List[str]] = None) -> None:
    """Register an action handler for specific connections or globally."""
    if connection_ids is None:
        self.global_actions[action_name] = handler
        self.logger.info(f"Registered global action: {action_name}")
    else:
        for conn_id in connection_ids:
            if conn_id not in self.actions:
                self.actions[conn_id] = {}
            self.actions[conn_id][action_name] = handler
        self.logger.info(f"Registered action {action_name} for connections: {connection_ids}")
remove_connection(connection_id) async

Remove a WebSocket connection from the pool.

Source code in toolboxv2/mods/WebSocketManager.py
73
74
75
76
77
async def remove_connection(self, connection_id: str) -> None:
    """Remove a WebSocket connection from the pool."""
    if connection_id in self.connections:
        del self.connections[connection_id]
        self.logger.info(f"Removed connection {connection_id} (remaining: {len(self.connections)})")
send_to_connection(connection_id, message) async

Send message to a specific connection.

Source code in toolboxv2/mods/WebSocketManager.py
144
145
146
147
148
149
150
151
152
153
async def send_to_connection(self, connection_id: str, message: str) -> bool:
    """Send message to a specific connection."""
    if connection_id in self.connections:
        try:
            await self.connections[connection_id].send(message)
            return True
        except Exception as e:
            self.logger.error(f"Failed to send to {connection_id}: {e}")
            await self.remove_connection(connection_id)
    return False

WebSocketServer

WebSocket server with pool management.

Source code in toolboxv2/mods/WebSocketManager.py
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
class WebSocketServer:
    """WebSocket server with pool management."""

    def __init__(self, host: str = "localhost", port: int = 8765):
        self.host = host
        self.port = port
        self.pools: Dict[str, WebSocketPool] = {}
        self.server = None
        self.logger = logging.getLogger("WSServer")

    def create_pool(self, pool_id: str) -> WebSocketPool:
        """Create a new connection pool."""
        if pool_id not in self.pools:
            self.pools[pool_id] = WebSocketPool(pool_id)
            self.logger.info(f"Created pool: {pool_id}")
        return self.pools[pool_id]

    def get_pool(self, pool_id: str) -> Optional[WebSocketPool]:
        """Get an existing pool."""
        return self.pools.get(pool_id)

    async def handle_connection(self, websocket, path: str):
        """Handle new WebSocket connections."""
        connection_id = f"conn_{id(websocket)}"
        pool_id = path.strip('/') or 'default'

        pool = self.create_pool(pool_id)
        await pool.add_connection(connection_id, websocket)

        self.logger.info(f"New connection {connection_id} in pool {pool_id}")

        try:
            # Ping-Task für diese Verbindung starten
            ping_task = asyncio.create_task(self._connection_ping_loop(websocket, connection_id))

            async for message in websocket:
                # Message handling in background to prevent blocking
                asyncio.create_task(pool.handle_message(connection_id, message))

        except ConnectionClosed:
            self.logger.info(f"Connection {connection_id} closed normally")
        except Exception as e:
            self.logger.error(f"Connection error for {connection_id}: {e}")
        finally:
            ping_task.cancel()
            await pool.remove_connection(connection_id)

    async def _connection_ping_loop(self, websocket, connection_id: str):
        """Ping loop for individual connection."""
        try:
            while not websocket.closed:
                await asyncio.sleep(30)  # Ping every 30 seconds
                await websocket.ping()
        except Exception as e:
            self.logger.debug(f"Ping loop ended for {connection_id}: {e}")

    async def start(self, non_blocking: bool = False) -> None:
        """Start the WebSocket server."""
        if non_blocking is None:
            return
        self.server = await ws_serve(self.handle_connection, self.host, self.port)
        self.logger.info(f"🚀 WebSocket server started on {self.host}:{self.port}")

        if not non_blocking:
            await self.server.wait_closed()

    async def stop(self) -> None:
        """Stop the server and close all connections."""
        if self.server:
            self.server.close()
            await self.server.wait_closed()

        # Close all pools
        for pool in self.pools.values():
            await pool.close_all()
        self.pools.clear()

        self.logger.info("✅ Server stopped")
create_pool(pool_id)

Create a new connection pool.

Source code in toolboxv2/mods/WebSocketManager.py
364
365
366
367
368
369
def create_pool(self, pool_id: str) -> WebSocketPool:
    """Create a new connection pool."""
    if pool_id not in self.pools:
        self.pools[pool_id] = WebSocketPool(pool_id)
        self.logger.info(f"Created pool: {pool_id}")
    return self.pools[pool_id]
get_pool(pool_id)

Get an existing pool.

Source code in toolboxv2/mods/WebSocketManager.py
371
372
373
def get_pool(self, pool_id: str) -> Optional[WebSocketPool]:
    """Get an existing pool."""
    return self.pools.get(pool_id)
handle_connection(websocket, path) async

Handle new WebSocket connections.

Source code in toolboxv2/mods/WebSocketManager.py
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
async def handle_connection(self, websocket, path: str):
    """Handle new WebSocket connections."""
    connection_id = f"conn_{id(websocket)}"
    pool_id = path.strip('/') or 'default'

    pool = self.create_pool(pool_id)
    await pool.add_connection(connection_id, websocket)

    self.logger.info(f"New connection {connection_id} in pool {pool_id}")

    try:
        # Ping-Task für diese Verbindung starten
        ping_task = asyncio.create_task(self._connection_ping_loop(websocket, connection_id))

        async for message in websocket:
            # Message handling in background to prevent blocking
            asyncio.create_task(pool.handle_message(connection_id, message))

    except ConnectionClosed:
        self.logger.info(f"Connection {connection_id} closed normally")
    except Exception as e:
        self.logger.error(f"Connection error for {connection_id}: {e}")
    finally:
        ping_task.cancel()
        await pool.remove_connection(connection_id)
start(non_blocking=False) async

Start the WebSocket server.

Source code in toolboxv2/mods/WebSocketManager.py
410
411
412
413
414
415
416
417
418
async def start(self, non_blocking: bool = False) -> None:
    """Start the WebSocket server."""
    if non_blocking is None:
        return
    self.server = await ws_serve(self.handle_connection, self.host, self.port)
    self.logger.info(f"🚀 WebSocket server started on {self.host}:{self.port}")

    if not non_blocking:
        await self.server.wait_closed()
stop() async

Stop the server and close all connections.

Source code in toolboxv2/mods/WebSocketManager.py
420
421
422
423
424
425
426
427
428
429
430
431
async def stop(self) -> None:
    """Stop the server and close all connections."""
    if self.server:
        self.server.close()
        await self.server.wait_closed()

    # Close all pools
    for pool in self.pools.values():
        await pool.close_all()
    self.pools.clear()

    self.logger.info("✅ Server stopped")

WhatsAppTb

client

DocumentSystem
Source code in toolboxv2/mods/WhatsAppTb/client.py
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
class DocumentSystem:
    def __init__(self, storage: BlobStorage):
        self.storage = storage
        self.media_types = {
            'document': ['pdf', 'doc', 'docx', 'txt'],
            'image': ['jpg', 'jpeg', 'png', 'gif'],
            'video': ['mp4', 'mov', 'avi']
        }

    def list_documents(self, filter_type: str = None) -> list[dict]:
        """List all documents with metadata"""
        docs = []
        for blob_id in self.storage._get_all_blob_ids():
            with BlobFile(blob_id, 'r', self.storage) as f:
                metadata = f.read_json()
                if metadata:
                    docs.append({
                        'id': blob_id,
                        'name': metadata.get('filename', blob_id),
                        'type': metadata.get('type', 'document'),
                        'size': metadata.get('size', 0),
                        'modified': metadata.get('timestamp', ''),
                        'preview': metadata.get('preview', '')
                    })
        if filter_type:
            return [d for d in docs if d['type'] == filter_type]
        return docs

    def save_document(self, file_data: bytes, filename: str, file_type: str) -> str:
        """Save a document with metadata"""
        blob_id = self.storage._generate_blob_id()
        metadata = {
            'filename': filename,
            'type': file_type,
            'size': len(file_data),
            'timestamp': datetime.now().isoformat(),
            'preview': self._generate_preview(file_data, file_type)
        }

        with BlobFile(blob_id, 'w', self.storage) as f:
            f.write_json(metadata)
            f.write(file_data)
        return blob_id

    def delete_document(self, blob_id: str) -> bool:
        """Delete a document"""
        try:
            self.storage.delete_blob(blob_id)
            return True
        except Exception as e:
            logging.error(f"Delete failed: {str(e)}")
            return False

    def search_documents(self, query: str) -> list[dict]:
        """Search documents by filename or content"""
        results = []
        for doc in self.list_documents():
            if query.lower() in doc['name'].lower() or self._search_in_content(doc['id'], query):
                results.append(doc)
        return results

    def _generate_preview(self, data: bytes, file_type: str) -> str:
        """Generate preview based on file type"""
        if file_type in self.media_types['image']:
            return f"Image preview: {data[:100].hex()}"
        elif file_type in self.media_types['video']:
            return "Video preview unavailable"
        return data[:100].decode('utf-8', errors='ignore')

    def _search_in_content(self, blob_id: str, query: str) -> bool:
        """Search content within documents"""
        try:
            with BlobFile(blob_id, 'r', self.storage) as f:
                content = f.read().decode('utf-8', errors='ignore')
                return query.lower() in content.lower()
        except:
            return False
delete_document(blob_id)

Delete a document

Source code in toolboxv2/mods/WhatsAppTb/client.py
112
113
114
115
116
117
118
119
def delete_document(self, blob_id: str) -> bool:
    """Delete a document"""
    try:
        self.storage.delete_blob(blob_id)
        return True
    except Exception as e:
        logging.error(f"Delete failed: {str(e)}")
        return False
list_documents(filter_type=None)

List all documents with metadata

Source code in toolboxv2/mods/WhatsAppTb/client.py
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
def list_documents(self, filter_type: str = None) -> list[dict]:
    """List all documents with metadata"""
    docs = []
    for blob_id in self.storage._get_all_blob_ids():
        with BlobFile(blob_id, 'r', self.storage) as f:
            metadata = f.read_json()
            if metadata:
                docs.append({
                    'id': blob_id,
                    'name': metadata.get('filename', blob_id),
                    'type': metadata.get('type', 'document'),
                    'size': metadata.get('size', 0),
                    'modified': metadata.get('timestamp', ''),
                    'preview': metadata.get('preview', '')
                })
    if filter_type:
        return [d for d in docs if d['type'] == filter_type]
    return docs
save_document(file_data, filename, file_type)

Save a document with metadata

Source code in toolboxv2/mods/WhatsAppTb/client.py
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
def save_document(self, file_data: bytes, filename: str, file_type: str) -> str:
    """Save a document with metadata"""
    blob_id = self.storage._generate_blob_id()
    metadata = {
        'filename': filename,
        'type': file_type,
        'size': len(file_data),
        'timestamp': datetime.now().isoformat(),
        'preview': self._generate_preview(file_data, file_type)
    }

    with BlobFile(blob_id, 'w', self.storage) as f:
        f.write_json(metadata)
        f.write(file_data)
    return blob_id
search_documents(query)

Search documents by filename or content

Source code in toolboxv2/mods/WhatsAppTb/client.py
121
122
123
124
125
126
127
def search_documents(self, query: str) -> list[dict]:
    """Search documents by filename or content"""
    results = []
    for doc in self.list_documents():
        if query.lower() in doc['name'].lower() or self._search_in_content(doc['id'], query):
            results.append(doc)
    return results
WhatsAppAssistant dataclass
Source code in toolboxv2/mods/WhatsAppTb/client.py
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
@dataclass
class WhatsAppAssistant:
    whc: WhClient
    isaa: 'Tools'
    agent: Optional['Agent'] = None
    credentials: Credentials | None = None
    state: AssistantState = AssistantState.OFFLINE

    # Service clients
    gmail_service: Any = None
    calendar_service: Any = None

    start_time: Any = None

    blob_docs_system: Any = None
    duration_minutes: int = 20
    credentials_path: str = "/root/Toolboxv2/credentials.json"
    # Progress messengers
    progress_messengers: dict[str, 'ProgressMessenger'] = field(default_factory=dict)
    buttons: dict[str, dict] = field(default_factory=dict)
    history: FileCache = field(default_factory=FileCache)

    pending_actions: dict[str, dict] = field(default_factory=dict)


    def __post_init__(self):

        self.start_time = datetime.now()
        self.processed_messages = set()
        self.message_lock = threading.Lock()
        self.audio_processor = None
        self.blob_docs_system = DocumentSystem(BlobStorage())
        self.stt = get_app().run_any(TBEF.AUDIO.STT_GENERATE,
                                     model="openai/whisper-small",
                                     row=False, device=1)

        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}

        self.load_credentials()
        self.setup_progress_messengers()
        self.setup_interaction_buttons()
        self.history = FileCache(folder=".data/WhatsAppAssistant")
        self.state = AssistantState.ONLINE

    async def generate_authorization_url(self, *a):
        """
        Generate an authorization URL for user consent

        :return: Authorization URL for the user to click and authorize access
        """
        from google_auth_oauthlib.flow import Flow
        # Define the scopes required for Gmail and Calendar
        SCOPES = [
            'https://www.googleapis.com/auth/gmail.modify',
            'https://www.googleapis.com/auth/calendar'
        ]

        # Create a flow instance to manage the OAuth 2.0 authorization process
        flow = Flow.from_client_secrets_file(
            self.credentials_path,
            scopes=SCOPES,
            redirect_uri='urn:ietf:wg:oauth:2.0:oob'  # Use 'urn:ietf:wg:oauth:2.0:oob' for desktop apps
        )

        # Generate the authorization URL
        authorization_url, _ = flow.authorization_url(
            access_type='offline',  # Allows obtaining refresh token
            prompt='consent'  # Ensures user is always prompted for consent
        )
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'auth',
                                                                              'step': 'awaiting_key'}
        return {
            'type': 'quick_reply',
            'text': f'Url to log in {authorization_url}',
            'options': {'cancel': '❌ Cancel Upload'}
        }

    def complete_authorization(self, message: Message):
        """
        Complete the authorization process using the authorization code

        :param authorization_code: Authorization code received from Google
        """
        from google_auth_oauthlib.flow import Flow
        authorization_code = message.content
        # Define the scopes required for Gmail and Calendar
        SCOPES = [
            'https://www.googleapis.com/auth/gmail.modify',
            'https://www.googleapis.com/auth/calendar'
        ]

        # Create a flow instance to manage the OAuth 2.0 authorization process
        flow = Flow.from_client_secrets_file(
            self.credentials_path,
            scopes=SCOPES,
            redirect_uri='urn:ietf:wg:oauth:2.0:oob'
        )

        # Exchange the authorization code for credentials
        flow.fetch_token(code=authorization_code)
        self.credentials = flow.credentials

        # Save the credentials for future use
        self.save_credentials()

        # Initialize services
        self.init_services()
        return "Done"


    def save_credentials(self):
        """
        Save the obtained credentials to a file for future use
        """
        if not os.path.exists('token'):
            os.makedirs('token')

        with open('token/google_token.json', 'w') as token_file:
            token_file.write(self.credentials.to_json())


    def load_credentials(self):
        """
        Load previously saved credentials if available

        :return: Whether credentials were successfully loaded
        """
        try:
            self.credentials = Credentials.from_authorized_user_file('token/google_token.json')
            self.init_services()
            return True
        except FileNotFoundError:
            return False


    def init_services(self):
        """
        Initialize Gmail and Calendar services
        """
        from googleapiclient.discovery import build

        self.gmail_service = build('gmail', 'v1', credentials=self.credentials)
        self.calendar_service = build('calendar', 'v3', credentials=self.credentials)
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}

    def setup_progress_messengers(self):
        """Initialize progress messengers for different types of tasks"""
        self.progress_messengers = {
            'task': self.whc.progress_messenger0,
            'email': self.whc.progress_messenger1,
            'calendar': self.whc.progress_messenger2
        }

    def setup_interaction_buttons(self):
        """Define WhatsApp interaction buttons for different functionalities"""
        self.buttons = {
            'menu': {
                'header': 'Digital Assistant',
                'body': 'Please select an option:',
                'footer': '-- + --',
                'action': {
                    'button': 'Menu',
                    'sections': [
                        {
                            'title': 'Main Functions',
                            'rows': [
                                {'id': 'agent', 'title': 'Agent Controls', 'description': 'Manage your AI assistant'},
                                {'id': 'email', 'title': 'Email Management', 'description': 'Handle your emails'},
                                {'id': 'calendar', 'title': 'Calendar', 'description': 'Manage your schedule'},
                                {'id': 'docs', 'title': 'Documents', 'description': 'Handle documents'},
                                {'id': 'system', 'title': 'System', 'description': 'System controls and metrics'}
                            ]
                        }
                    ]
                }
            },
            'agent': self._create_agent_controls_buttons(),
            'email': self._create_email_controls_buttons(),
            'calendar': self._create_calendar_controls_buttons(),
            'docs': self._create_docs_controls_buttons(),
            'system': self._create_system_controls_buttons()
        }

    @staticmethod
    def _create_agent_controls_buttons():
        return {
            'header': 'Agent Controls',
            'body': 'Manage your AI assistant:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'agent-task', 'title': 'Agent Task', 'description': 'Run the agent'},
                            {'id': 'start', 'title': 'Start Agent', 'description': 'Run taskstack in background'},
                            {'id': 'stop', 'title': 'Stop Agent', 'description': 'Stop taskstack execution'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'system-task', 'title': 'System Task',
                             'description': 'Run the Isaa Reasoning Agent system'},
                            {'id': 'tasks', 'title': 'Task Stack', 'description': 'View and manage tasks'},
                            {'id': 'memory', 'title': 'Clear Memory', 'description': 'Reset agent memory'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_email_controls_buttons():
        return {
            'header': 'Email Management',
            'body': 'Handle your emails:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'check', 'title': 'Check Emails', 'description': 'View recent emails'},
                            {'id': 'send', 'title': 'Send Email', 'description': 'Compose new email'},
                            {'id': 'summary', 'title': 'Get Summary', 'description': 'Summarize emails'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'search', 'title': 'Search', 'description': 'Search emails'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_calendar_controls_buttons():
        return {
            'header': 'Calendar Management',
            'body': 'Manage your schedule:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'today', 'title': 'Today\'s Events', 'description': 'View today\'s schedule'},
                            {'id': 'add', 'title': 'Add Event', 'description': 'Create new event'},
                            {'id': 'upcoming', 'title': 'Upcoming', 'description': 'View upcoming events'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'find_slot', 'title': 'Find Time Slot', 'description': 'Find available time'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_docs_controls_buttons():
        return {
            'header': 'Document Management',
            'body': 'Handle your documents:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'upload', 'title': 'Upload', 'description': 'Add new document'},
                            {'id': 'list', 'title': 'List Documents', 'description': 'View all documents'},
                            {'id': 'search', 'title': 'Search', 'description': 'Search documents'}
                        ]
                    },
                    {
                        'title': 'Advanced Actions',
                        'rows': [
                            {'id': 'delete', 'title': 'Delete', 'description': 'Remove document'}
                        ]
                    }
                ]
            }
        }

    @staticmethod
    def _create_system_controls_buttons():
        return {
            'header': 'System Controls',
            'body': 'System management:',
            'action': {
                'button': 'Select',
                'sections': [
                    {
                        'title': 'Basic Actions',
                        'rows': [
                            {'id': 'status', 'title': 'System Status', 'description': 'View current status'},
                            {'id': 'restart', 'title': 'Restart', 'description': 'Restart system'},
                            {'id': 'connect', 'title': 'Connect', 'description': 'Connect to Google Calendar and Email'}
                        ]
                    }
                ]
            }
        }

    async def handle_message(self, message: 'Message'):
        """Main message handler for incoming WhatsApp messages"""

        # Deduplication check
        with self.message_lock:
            if message.id in self.processed_messages:
                return
            last_ts = time.time()
            print(last_ts)
            if len(self.processed_messages) > 0:
                m_id, last_ts = self.processed_messages.pop()
                self.processed_messages.add((m_id, last_ts))

            print("DUPLICATION P", message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0) , last_ts)
            if float(message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0)) < last_ts - 120:
                return
            self.processed_messages.add((message.id, time.perf_counter()))

        # Mark message as read
        message.mark_as_read()

        # Extract content and type
        content_type = message.type
        content = message.content

        print(f"message.content {content=} {content_type=} {message.data=}")

        try:
            if content_type == 'interactive':
                await self.handle_interactive(message)
            elif content_type == 'audio':
                await self.handle_audio_message(message)
            elif content_type in ['document', 'image', 'video']:
                response = await self.handle_media_message(message)
                self.save_reply(message, response)
            elif content_type == 'text':
                if content.lower() == "menu":
                    self.whc.messenger.send_button(
                        recipient_id=self.whc.progress_messenger0.recipient_phone,
                        button=self.buttons[content.lower()]
                    )
                else:
                    await self.helper_text(message)
            else:
                message.reply("Unsupported message type")
        #except Exception as e:
        #    logging.error(f"Message handling error: {str(e)}")
        #   message.reply("❌ Error processing request")
        finally:
            # Cleanup old messages (keep 1 hour history)
            with self.message_lock:
                self._clean_processed_messages()

    async def helper_text(self, message: 'Message', return_text=False):
        if not isinstance(message.content, str) and not len(message.content) > 0:
            content = self.whc.messenger.get_message(message.data)
            print(f"contents {content=}, {message.content=}")
            message.content = content
        self.history.set(message.id, message.content)
        if len(self.pending_actions[self.whc.progress_messenger0.recipient_phone].keys()) != 0:
            message.reply(
                f"Open Interaction : {json.dumps(self.pending_actions[self.whc.progress_messenger0.recipient_phone], indent=2)}")
            if self.pending_actions[self.whc.progress_messenger0.recipient_phone].get('type') == 'auth':
                res = self.complete_authorization(message)
                self.save_reply(message, res)
            res = await self.handle_calendar_actions(message)
            if res:
                self.save_reply(message, res)
                return
            res2 = await self.handle_email_actions(message)
            if res2:
                self.save_reply(message, res2)
                return
            await self.handle_agent_actions(message)
            return
        await self.handle_agent_actions(message)

    async def handle_interactive(self, message: Message):
        """Handle all interactive messages"""
        content = self.whc.messenger.get_interactive_response(message.data)
        if content.get("type") == "list_reply":
            await self.handle_button_interaction(content.get("list_reply"), message)
        elif content.get("type") == "button_reply":
            print(content)

    async def handle_audio_message(self, message: 'Message'):
        """Process audio messages with STT and TTS"""
        # Download audio
        progress = self.progress_messengers['task']
        stop_flag = threading.Event()
        # message_id = progress.send_initial_message(mode="loading")
        progress.message_id = message.id
        progress.start_loading_in_background(stop_flag)

        content = self.whc.messenger.get_audio(message.data)
        audio_file_name = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')), mime_type='audio/opus', file_path=".data/temp")
        print(f"audio_file_name {audio_file_name}")
        if audio_file_name is None:
            message.reply("Could not process audio file")
            stop_flag.set()
            return

        text = self.stt(audio_file_name)['text']
        if not text:
            message.reply("Could not process audio")
            stop_flag.set()
            return

        message.reply("Transcription :\n "+ text)
        message.content = text
        agent_res = await self.helper_text(message, return_text=True)

        if agent_res is not None:
            pass

        stop_flag.set()
        # Process text and get response
        # response = await self.process_input(text, message)

        # Convert response to audio
        #audio_file = self.audio_processor.tts(response)
        #audio_file = None # TODO
        #self.whc.messenger.send_audio(
        #    audio=audio_file,
        #    recipient_id=self.whc.progress_messenger0.recipient_phone,
        #)

    async def confirm(self, message: Message):
        status = self.pending_actions[self.whc.progress_messenger0.recipient_phone]
        if status.get('type') == "create_event":
            if status.get('step') == "confirm_envet":
                event = self._create_calendar_event(status.get('event_data'))
                self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                return f"✅ Event created!\n{event.get('htmlLink')}"
            return "❌"
        elif status.get('type') == "compose_email":
            if status.get('step') == "confirm_email":
                # Send email
                result = self.gmail_service.users().messages().send(
                    userId='me',
                    body=self._build_email_draft(status['draft'])
                ).execute()
                self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                return f"✅ Email sent! Message ID: {result['id']}"
            return "❌"
        return "❌ Done"

    async def cancel(self, *a):
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
        return "✅ cancel Done"

    async def handle_button_interaction(self, content: dict, message: Message):
        """Handle button click interactions"""
        button_id = content['id']

        # First check if it's a main menu button
        if button_id in self.buttons:
            self.whc.messenger.send_button(
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                button=self.buttons[button_id]
            )
            return

        # Handle action buttons
        action_handlers = {
            # Agent controls
            'start': self.start_agent,
            'stop': self.stop_agent,
            'tasks': self.show_task_stack,
            'memory': self.clear_memory,
            'system-task': self.system_task,
            'agent-task': self.agent_task,

            # Email controls
            'check': self.check_emails,
            'send': self.start_email_compose,
            'summary': self.email_summary,
            'search': self.email_search,

            # Calendar controls
            'today': self.show_today_events,
            'add': self.start_event_create,
            'upcoming': self.show_upcoming_events,
            'find_slot': self.find_time_slot,

            # Document controls
            'upload': self.start_document_upload,
            'list': self.list_documents,
            'search_docs': self.search_documents,
            'delete': self.delete_document,

            # System controls
            'status': self.system_status,
            'restart': self.restart_system,
            'connect': self.generate_authorization_url,

            'cancel': self.cancel,
            'confirm': self.confirm,
        }
        if button_id in action_handlers:
            try:
                # Start progress indicator
                progress = self.progress_messengers['task']
                stop_flag = threading.Event()
                # message_id = progress.send_initial_message(mode="loading")
                progress.message_id = message.id
                progress.start_loading_in_background(stop_flag)

                # Execute handler

                result = await action_handlers[button_id](message)


                # Send result
                if isinstance(result, str):
                    self.save_reply(message, result)
                elif isinstance(result, dict):  # For structured responses
                    self.send_structured_response(result)

                stop_flag.set()
            finally:
                #except Exception as e:
                stop_flag.set()
            #    message.reply(f"❌ Error processing {button_id}: {str(e)}")
        elif 'event_' in button_id:
            res = await self.get_event_details(button_id.replace("event_", ''))
            if isinstance(res, str):
                self.save_reply(message, res)
                return
            for r in res:
                if isinstance(r, str):
                    self.save_reply(message, r)
                else:
                    self.whc.messenger.send_location(**r)

        elif 'email_' in button_id:
            res = await self.get_email_details(button_id.replace("email_", ''))
            self.save_reply(message, res)
        else:
            message.reply("⚠️ Unknown command")

    def send_structured_response(self, result: dict):
        """Send complex responses using appropriate WhatsApp features"""
        if result['type'] == 'list':
            self.whc.messenger.send_button(
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                button={
                    'header': result.get('header', ''),
                    'body': result.get('body', ''),
                    'footer': result.get('footer', ''),
                    'action': {
                        'button': 'Action',
                        'sections': result['sections']
                    }
                }
            )
        elif result['type'] == 'quick_reply':
            self.whc.messenger.send_button(
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                button={
                    'header': "Quick reply",
                    'body': result['text'],
                    'footer': '',
                    'action': {'button': 'Action', 'sections': [{
                        'title': 'View',
                        'rows': [{'id': k, 'title': v[:23]} for k, v in result['options'].items()]
                    }]}
                }
            )

        elif result['type'] == 'media':
            if result['media_type'] == 'image':
                self.whc.messenger.send_image(
                    image=result['url'],
                    recipient_id=self.whc.progress_messenger0.recipient_phone,
                    caption=result.get('caption', '')
                )
            elif result['media_type'] == 'document':
                self.whc.messenger.send_document(
                    document=result['url'],
                    recipient_id=self.whc.progress_messenger0.recipient_phone,
                    caption=result.get('caption', '')
                )

    async def clear_memory(self, message):
        self.agent.reset_context()
        self.agent.taskstack.tasks = []
        return "🧠 Memory cleared successfully"

    async def system_task(self, message):
        """Initiate email search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'system',
            'step': 'await_query'
        }
        return {
            'type': 'quick_reply',
            'text': "Now prompt the 🧠ISAA-System 📝",
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def agent_task(self, message):
        """Initiate email search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'self-agent',
            'step': 'await_query'
        }
        return {
            'type': 'quick_reply',
            'text': "Now prompt the self-agent 📝",
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def check_emails(self, message, query=""):
        """Improved email checking with WhatsApp API formatting"""
        if not self.gmail_service:
            return "⚠️ Gmail service not configured"

        try:
            results = self.gmail_service.users().messages().list(
                userId='me',
                maxResults=10,
                labelIds=['INBOX'],
                q=query
            ).execute()

            emails = []
            for msg in results.get('messages', [])[:10]:
                email_data = self.gmail_service.users().messages().get(
                    userId='me',
                    id=msg['id'],
                    format='metadata'
                ).execute()

                headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
                emails.append({
                    'id': msg['id'],
                    'from': headers.get('From', 'Unknown'),
                    'subject': headers.get('Subject', 'No Subject'),
                    'date': headers.get('Date', 'Unknown'),
                    'snippet': email_data.get('snippet', ''),
                    'unread': 'UNREAD' in email_data.get('labelIds', [])
                })

            return {
                'type': 'list',
                'header': '📨 Recent Emails',
                'body': 'Tap to view full email',
                'footer': 'Email Manager',
                'sections': [{
                    'title': f"Inbox ({len(emails)} emails)",
                    'rows': [{
                        'id': f"email_{email['id']}",
                        'title': f"{'📬' if email['unread'] else '📭'} {email['subject']}"[:23],
                        'description': f"From: {email['from']}\n{email['snippet']}"[:45]
                    } for email in emails]
                }]
            }
        except Exception as e:
            return f"⚠️ Error fetching emails: {str(e)}"

    async def get_email_details(self, email_id):
        """Retrieve and format full email details"""
        if not self.gmail_service:
            return "⚠️ Gmail service not configured"

        try:
            email_data = self.gmail_service.users().messages().get(
                userId='me',
                id=email_id,
                format='full'
            ).execute()

            headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
            body = ""
            for part in email_data.get('payload', {}).get('parts', []):
                if part['mimeType'] == 'text/plain':
                    body = base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
                    break

            formatted_text = (
                f"📧 *Email Details*\n\n"
                f"From: {headers.get('From', 'Unknown')}\n"
                f"Subject: {headers.get('Subject', 'No Subject')}\n"
                f"Date: {headers.get('Date', 'Unknown')}\n\n"
                f"{body[:15000]}{'...' if len(body) > 15000 else ''}"
            )
            return  self.agent.mini_task(
                formatted_text , "system", "Summarize the email in bullet points with key details"
            )
        except Exception as e:
            return f"⚠️ Error fetching email: {str(e)}"

    async def email_summary(self, message):
        """Generate AI-powered email summaries"""
        try:
            messages = self.gmail_service.users().messages().list(
                userId='me',
                maxResults=3,
                labelIds=['INBOX']
            ).execute().get('messages', [])

            email_contents = []
            for msg in messages[:3]:
                email_data = self.gmail_service.users().messages().get(
                    userId='me',
                    id=msg['id'],
                    format='full'
                ).execute()
                email_contents.append(self._parse_email_content(email_data))

            summary = self.agent.mini_task(
                "\n\n".join(email_contents) , "system", "Summarize these emails in bullet points with key details:"
            )

            return f"📋 Email Summary:\n{summary}\n\n*Powered by AI*"
        except Exception as e:
            logging.error(f"Summary failed: {str(e)}")
            return f"❌ Could not generate summary: {str(e)}"

    async def email_search(self, message):
        """Initiate email search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'email_search',
            'step': 'await_query'
        }
        return {
            'type': 'quick_reply',
            'text': "🔍 What would you like to search for?",
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def start_email_compose(self, message):
        """Enhanced email composition workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'compose_email',
            'step': 'subject',
            'draft': {'attachments': []}
        }
        return {
            'type': 'quick_reply',
            'text': "📝 Let's compose an email\n\nSubject:",
            'options': {'cancel': '❌ Cancel Composition'}
        }

    async def handle_email_actions(self, message):
        """Handle multi-step email workflows"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('type') == 'compose_email':
            return await self._handle_email_composition(message, user_state)
        if user_state.get('type') == 'email_search':
            return await self.check_emails(message, self.agent.mini_task("""Conventire Pezise zu einer googel str only query using : Gmail Suchoperatoren!

Basis-Operatoren:
- from: Absender
- to: Empfänger
- subject: Betreff
- label: Gmail Label
- has:attachment Anhänge
- newer_than:7d Zeitfilter
- before: Datum vor
- after: Datum nach

Erweiterte Operatoren:
- in:inbox
- in:sent
- in:spam
- cc: Kopie
- bcc: Blindkopie
- is:unread
- is:read
- larger:10M Größenfilter
- smaller:5M
- filename:pdf Dateityp

Profi-Tipps:
- Kombinierbar mit UND/ODER
- Anführungszeichen für exakte Suche
- Negation mit -
 beispeile : 'Ungelesene Mails letzte Woche': -> 'is:unread newer_than:7d'

""", "user",message.content))


        return None

    async def _handle_email_composition(self, message, state):
        if state['step'] == 'subject':
            state['draft']['subject'] = message.content
            state['step'] = 'body'
            return {
                'type': 'quick_reply',
                'text': "✍️ Email body:",
                'options': {'attach': '📎 Add Attachment', 'send': '📤 Send Now'}
            }

        elif state['step'] == 'body':
            if message.content == 'attach':
                state['step'] = 'attachment'
                return "📎 Please send the file you want to attach"

            state['draft']['body'] = message.content
            state['step'] = 'confirm_email'
            return {
                'type': 'quick_reply',
                'text': f"📧 Ready to send?\n\nSubject: {state['draft']['subject']}\n\n{state['draft']['body']}",
                'options': {'confirm': '✅ Send', 'cancel': '❌ cancel'}
            }

        elif state['step'] == 'attachment':
            # Handle attachment upload
            file_type = message.type
            if file_type not in ['document', 'image']:
                return "❌ Unsupported file type"

            media_url = getattr(message, file_type).id
            media_data = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=media_url), mime_type=media_url.type, file_path=".data/temp")
            state['draft']['attachments'].append(media_data)
            state['step'] = 'body'
            return "📎 Attachment added! Add more or send the email"


    def _parse_email_content(self, email_data):
        """Extract readable content from email payload"""
        parts = email_data.get('payload', {}).get('parts', [])
        body = ""
        for part in parts:
            if part['mimeType'] == 'text/plain':
                body += base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
        return f"Subject: {email_data.get('subject', '')}\nFrom: {email_data.get('from', '')}\n\n{body}"

    def _build_email_draft(self, draft):
        """Create MIME message from draft data"""
        message = MIMEMultipart()
        message['to'] = draft.get('to', '')
        message['subject'] = draft['subject']
        message.attach(MIMEText(draft['body']))

        for attachment in draft['attachments']:
            part = MIMEBase('application', 'octet-stream')
            part.set_payload(attachment)
            encoders.encode_base64(part)
            part.add_header('Content-Disposition', 'attachment')
            message.attach(part)

        return {'raw': base64.urlsafe_b64encode(message.as_bytes()).decode()}

    def _get_email_subject(self, msg):
        headers = msg.get('payload', {}).get('headers', [])
        return next((h['value'] for h in headers if h['name'] == 'Subject'), 'No Subject')

    def _get_email_sender(self, msg):
        headers = msg.get('payload', {}).get('headers', [])
        return next((h['value'] for h in headers if h['name'] == 'From'), 'Unknown Sender')

    def _get_email_snippet(self, msg):
        return msg.get('snippet', '')[:100] + '...'
    # Calendar Handlers

    # Calendar Functions
    def _format_event_time(self, event):
        """Improved time formatting for calendar events"""
        start = event['start'].get('dateTime', event['start'].get('date'))
        end = event['end'].get('dateTime', event['end'].get('date'))

        try:
            start_dt = parser.parse(start)
            end_dt = parser.parse(end)
            if 'T' in start:
                return f"{start_dt.strftime('%a %d %b %H:%M')} - {end_dt.strftime('%H:%M')}"
            return f"{start_dt.strftime('%d %b %Y')} (All Day)"
        except:
            return "Time not specified"

    async def get_event_details(self, event_id):
        """Retrieve and format calendar event details with location support"""
        if not self.calendar_service:
            return "⚠️ Calendar service not configured"

        try:
            event = self.calendar_service.events().get(
                calendarId='primary',
                eventId=event_id
            ).execute()

            response = [ (
                    f"📅 *Event Details*\n\n"
                    f"Title: {event.get('summary', 'No title')}\n"
                    f"Time: {self._format_event_time(event)}\n"
                    f"Location: {event.get('location', 'Not specified')}\n\n"
                    f"{event.get('description', 'No description')[:1000]}"
                )]

            if 'geo' in event:
                response.append({
                    'lat': float(event['geo']['latitude']),
                    'long': float(event['geo']['longitude']),
                    'name': event.get('location', 'Event Location'),
                    'address': event.get('location', ''),
                    'recipient_id': self.whc.progress_messenger0.recipient_phone
                })
            return response
        except Exception as e:
            return f"⚠️ Error fetching event: {str(e)}"

    async def show_today_events(self, message):
        """Show today's calendar events"""
        if not self.calendar_service:
            message.replay("service not online")

        now = datetime.utcnow().isoformat() + 'Z'
        end_of_day = (datetime.now() + timedelta(days=1)).replace(
            hour=0, minute=0, second=0).isoformat() + 'Z'

        events_result = self.calendar_service.events().list(
            calendarId='primary',
            timeMin=now,
            timeMax=end_of_day,
            singleEvents=True,
            orderBy='startTime'
        ).execute()

        events = events_result.get('items', [])
        return self._format_calendar_response(events, "Today's Events")

    # Updated Calendar List Handlers
    async def show_upcoming_events(self, message):
        """Show upcoming events with interactive support"""
        if not self.calendar_service:
            return "⚠️ Calendar service not configured"

        try:
            now = datetime.utcnow().isoformat() + 'Z'
            next_week = (datetime.now() + timedelta(days=7)).isoformat() + 'Z'

            events_result = self.calendar_service.events().list(
                calendarId='primary',
                timeMin=now,
                timeMax=next_week,
                singleEvents=True,
                orderBy='startTime',
                maxResults=10
            ).execute()

            events = events_result.get('items', [])
            return self._format_calendar_response(events, "Upcoming Events")
        except Exception as e:
            return f"⚠️ Error fetching events: {str(e)}"

    async def start_event_create(self, message):
        """Initiate event creation workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
            'type': 'create_event',
            'step': 'title',
            'event_data': {}
        }
        return {
            'type': 'quick_reply',
            'text': "Let's create an event! What's the title?",
            'options': {'cancel': '❌ Cancel'}
        }

    async def find_time_slot(self, message):
        """Find and display the next 5 available time slots with dynamic durations"""
        if not self.calendar_service:
            return "⚠️ Calendar service not configured"

        try:
            # Define the time range for the search (next 24 hours)
            now = datetime.now(UTC)
            end_time = now + timedelta(days=1)

            # FreeBusy Request
            freebusy_request = {
                "timeMin": now.isoformat(),
                "timeMax": end_time.isoformat(),
                "items": [{"id": 'primary'}]
            }

            freebusy_response = self.calendar_service.freebusy().query(body=freebusy_request).execute()
            busy_slots = freebusy_response['calendars']['primary']['busy']

            # Slot-Berechnung
            available_slots = self._calculate_efficient_slots(
                busy_slots,
                self.duration_minutes
            )

            # Format the response for WhatsApp
            return {
                'type': 'list',
                'header': "⏰ Available Time Slots",
                'body': "Tap to select a time slot",
                'footer': "Time Slot Finder",
                'sections': [{
                    'title': "Next 5 Available Slots",
                    'rows': [{
                        'id': f"slot_{slot['start'].timestamp()}",
                        'title': f"🕒 {slot['start'].strftime('%H:%M')} - {slot['end'].strftime('%H:%M')}",
                        'description': f"Duration: {slot['duration']}"
                    } for slot in available_slots[:5]]
                }]
            }
        except Exception as e:
            return f"⚠️ Error finding time slots: {str(e)}"

    def _calculate_efficient_slots(self, busy_slots, duration_minutes):
        """Effiziente Slot-Berechnung"""
        available_slots = []
        current = datetime.now(UTC)
        end_time = current + timedelta(days=1)

        while current < end_time:
            slot_end = current + timedelta(minutes=duration_minutes)

            if slot_end > end_time:
                break

            is_available = all(
                slot_end <= parser.parse(busy['start']) or
                current >= parser.parse(busy['end'])
                for busy in busy_slots
            )

            if is_available:
                available_slots.append({
                    'start': current,
                    'end': slot_end,
                    'duration': f"{duration_minutes} min"
                })
                current = slot_end
            else:
                current += timedelta(minutes=15)

        return available_slots

    async def handle_calendar_actions(self, message):
        """Handle calendar-related pending actions"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('type') == 'create_event':
            return await self._handle_event_creation(message, user_state)

        return None

    async def _handle_event_creation(self, message, state):
        step = state['step']
        event_data = state['event_data']

        if step == 'title':
            event_data['summary'] = message.content
            state['step'] = 'start_time'
            return "📅 When should it start? (e.g., 'tomorrow 2pm' or '2024-03-20 14:30')"

        elif step == 'start_time':
            event_data['start'] = self._parse_time(message.content)
            state['step'] = 'end_time'
            return "⏰ When should it end? (e.g., '3pm' or '2024-03-20 15:30')"

        elif step == 'end_time':
            event_data['end'] = self._parse_time(message.content, reference=event_data['start'])
            state['step'] = 'description'
            return "📝 Add a description (or type 'skip')"

        elif step == 'description':
            if message.content.lower() != 'skip':
                event_data['description'] = message.content
            state['step'] = 'confirm_envet'
            return self._create_confirmation_message(event_data)

    def _format_calendar_response(self, events, title):
        """Enhanced calendar formatting with interactive support"""
        if not events:
            return f"📅 No {title.lower()} found"

        return {
            'type': 'list',
            'header': title,
            'body': "Tap to view event details",
            "footer": "-- Calendar --",
            'sections': [{
                'title': f"{len(events)} Events",
                'rows': [{
                    'id': f"event_{event['id']}",
                    'title': f"📅 {event['summary']}"[:23],
                    'description': self._format_event_time(event)[:45]
                } for event in events[:5]]
            }]
        }

    def _parse_iso_to_readable(self, iso_str):
        """Convert ISO datetime to readable format"""
        dt = datetime.fromisoformat(iso_str.replace('Z', '+00:00'))
        return dt.strftime("%a %d %b %Y %H:%M")

    def _parse_time(self, time_str, reference=None):
        """
        Konvertiert natürliche Sprache zu präziser Datetime

        Unterstützt:
        - 'heute'
        - 'morgen'
        - 'in einer woche'
        - '10 uhr'
        - '10pm'
        - 'nächsten montag'
        """
        if reference is None:
            reference = datetime.now()

        try:
            import dateparser

            # Dateparser für flexibel Zeitparsing
            parsed_time = dateparser.parse(
                time_str,
                settings={
                    'PREFER_DATES_FROM': 'future',
                    'RELATIVE_BASE': reference,
                    'TIMEZONE': 'Europe/Berlin'
                }
            )

            if parsed_time is None:
                # Fallback auf dateutil wenn dateparser scheitert
                parsed_time = parser .parse(time_str, fuzzy=True, default=reference)

            return parsed_time

        except Exception as e:
            print(f"Zeitparsing-Fehler: {e}")
            return reference

    def _calculate_free_slots(self, start, end, busy_slots):
        """Calculate free time slots between busy periods"""
        # Implementation would calculate available windows
        return [{
            'start': "09:00",
            'end': "11:00",
            'duration': "2 hours"
        }]

    def _create_confirmation_message(self, event_data):
        """Create event confirmation message"""
        details = [
            f"📌 Title: {event_data['summary']}",
            f"🕒 Start: {self._parse_iso_to_readable(event_data['start'])}",
            f"⏰ End: {self._parse_iso_to_readable(event_data['end'])}",
            f"📝 Description: {event_data.get('description', 'None')}"
        ]
        return {
            'type': 'quick_reply',
            'text': "\n".join(details),
            'options': {'confirm': '✅ Confirm', 'cancel': '❌ Cancel'}
        }

    def _create_calendar_event(self, event_data):
        """Create event through Calendar API"""
        event = {
            'summary': event_data['summary'],
            'start': {'dateTime': event_data['start']},
            'end': {'dateTime': event_data['end']},
        }
        if 'description' in event_data:
            event['description'] = event_data['description']

        return self.calendar_service.events().insert(
            calendarId='primary',
            body=event
        ).execute()

    async def system_status(self, message):
        o = (datetime.now() - self.start_time)
        o.microseconds = 0
        status = {
            "🤖 Agent": "Online" if self.agent else "Offline",
            "📧 Email": "Connected" if self.gmail_service else "Disconnected",
            "📅 Calendar": "Connected" if self.calendar_service else "Disconnected",
            "📄 Documents": "Connected" if self.blob_docs_system else "Disconnected",
            "⏳ Uptime": f"{str(o.isoformat())}"
        }
        return "\n".join([f"{k}: {v}" for k, v in status.items()])

    async def restart_system(self, message):
        message.reply("🔄 System restart initiated...")
        time.sleep(1)
        await self.clear_memory(message)
        time.sleep(1)
        return  "✅ System restarted"

    # Updated document handlers
    async def list_documents(self, message, filter_type=None):
        docs = self.blob_docs_system.list_documents(filter_type)
        if len(docs) == 0:
            return "No docs found"
        else:
            return str(docs)
        return {
            'type': 'list',
            'body': 'Stored Documents',
            'action': {
                'sections': [{
                    'title': 'Your Documents',
                    'rows': [{
                        'id': doc['id'],
                        'title': f"{self._get_icon(doc['type'])} {doc['name']}"[:23],
                        'description': f"{doc['type'].title()} | {self._format_size(doc['size'])} | {doc['modified']}"[:29]
                    } for doc in docs[:10]]
                }]}
        }

    async def start_document_upload(self, message):
        """Initiate document upload workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'document', 'step': 'awaiting_file'}
        return {
            'type': 'quick_reply',
            'text': '📤 Send me the file you want to upload',
            'options': {'cancel': '❌ Cancel Upload'}
        }

    async def search_documents(self, message):
        """Initiate document search workflow"""
        self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'search', 'step': 'awaiting_query'}
        return {
            'type': 'quick_reply',
            'text': '🔍 What are you looking for?',
            'options': {'cancel': '❌ Cancel Search'}
        }

    async def handle_media_message(self, message: 'Message'):
        """Handle document/image/video uploads"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('step') == 'awaiting_file':
            file_type = message.type
            if file_type not in ['document', 'image', 'video']:
                return "Unsupported file type"

            try:
                # Download media
                #media_url = message.document.url if hasattr(message, 'document') else \
                #    message.image.url if hasattr(message, 'image') else \
                #        message.video.url
                if file_type =='video':
                    content = self.whc.messenger.get_video(message.data)
                if file_type =='image':
                    content = self.whc.messenger.get_image(message.data)
                if file_type =='document':
                    content = self.whc.messenger.get_document(message.data)
                print("Media content:", content)
                media_data = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')),  mime_type=content.get('mime_type'), file_path='.data/temp')
                print("Media media_data:", media_data)
                # Save to blob storage
                filename = f"file_{file_type}_{datetime.now().isoformat()}_{content.get('sha256', '')}"
                blob_id = self.blob_docs_system.save_document(
                    open(media_data, 'rb').read(),
                    filename=filename,
                    file_type=file_type
                )

                self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                return f"✅ File uploaded successfully!\nID: {blob_id}"

            except Exception as e:
                logging.error(f"Upload failed: {str(e)}")
                return f"❌ Failed to upload file Error : {str(e)}"

        return "No pending uploads"

    async def delete_document(self, message):
        """Delete document workflow"""
        docs = self.blob_docs_system.list_documents()
        return {
            'type': 'quick_reply',
            'text': 'Select document to delete:',
            'options': {doc['id']: doc['name'] for doc in docs[:5]},
            'handler': self._confirm_delete
        }

    async def _confirm_delete(self, doc_id, message):
        """Confirm deletion workflow"""
        doc = next((d for d in self.blob_docs_system.list_documents() if d['id'] == doc_id), None)
        if not doc:
            return "Document not found"

        if self.blob_docs_system.delete_document(doc_id):
            return f"✅ {doc['name']} deleted successfully"
        return "❌ Failed to delete document"

    # Helper methods
    def _get_icon(self, file_type: str) -> str:
        icons = {
            'document': '📄',
            'image': '🖼️',
            'video': '🎥'
        }
        return icons.get(file_type, '📁')

    def _format_size(self, size: int) -> str:
        if size < 1024:
            return f"{size}B"
        elif size < 1024 ** 2:
            return f"{size / 1024:.1f}KB"
        elif size < 1024 ** 3:
            return f"{size / (1024 ** 2):.1f}MB"
        return f"{size / (1024 ** 3):.1f}GB"

    # Utility Methods

    def _clean_processed_messages(self):
        """Clean old messages from processed cache"""
        now = time.time()
        self.processed_messages = {
            msg_id for msg_id, timestamp in self.processed_messages
            if now - timestamp < 3600  # 1 hour retention
        }

    def send_email(self, to, subject, body):
        """Actual email sending function to be called by agent"""
        if not self.gmail_service:
            return False

        message = MIMEText(body)
        message['to'] = to
        message['subject'] = subject

        encoded_message = base64.urlsafe_b64encode(message.as_bytes()).decode()
        self.gmail_service.users().messages().send(
            userId='me',
            body={'raw': encoded_message}
        ).execute()
        return True

    async def start_agent(self, *a):
        """Start the agent in background mode"""
        if self.agent:
            self.agent.run_in_background()
            return True
        return False

    async def stop_agent(self, *b):
        """Stop the currently running agent"""
        if self.agent:
            self.agent.stop()
            return True
        return False

    async def show_task_stack(self, *a):
        """Display current task stack"""
        if self.agent and len(self.agent.taskstack.tasks) > 0:
            tasks = self.agent.taskstack.tasks
            return self.agent.mini_task("\n".join([f"Task {t.id}: {t.description}" for t in tasks]), "system", "Format to nice and clean whatsapp format")
        return "No tasks in stack"

    def run(self):
        """Start the WhatsApp assistant"""
        try:
            self.state = AssistantState.ONLINE
            # Send welcome message

            mas = self.whc.messenger.create_message(
                content="Digital Assistant is online! Send /help for available commands.",to=self.whc.progress_messenger0.recipient_phone,
            ).send(sender=0)
            mas_id = mas.get("messages", [{}])[0].get("id")
            print(mas_id)

        except Exception as e:
            logging.error(f"Assistant error: {str(e)}")
            self.state = AssistantState.OFFLINE
            raise

    async def handle_agent_actions(self, message):
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})
        def helper():

            stop_flag = threading.Event()
            try:
                progress = self.progress_messengers['task']
                # message_id = progress.send_initial_message(mode="loading")
                progress.message_id = message.id
                progress.start_loading_in_background(stop_flag)
                res = message.content
                print(message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get(
                    'context'))
                if context := message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get(
                    'context'):
                    context_str = f"Context : source {'USER' if context.get('from') in self.whc.progress_messenger0.recipient_phone else 'AGENT'}"
                    cd = self.history.get(context.get('id'))
                    context_str += "\n" + (cd if cd is not None else "The ref Message is not in the history")
                    res += "\n" + context_str
                if user_state.get('type') == 'system':
                    res = self.isaa.run(res)
                    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                elif user_state.get('type') == 'self-agent':
                    res = self.agent.run(res)
                    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
                self.agent.mode = LLMMode(
                    name="Chatter",
                    description="whatsapp Chat LLM",
                    system_msg="Response precise and short style using whatsapp syntax!",
                    post_msg=None
                )
                response = self.agent.mini_task(res, "user", persist=True)
                self.save_reply(message, response)
            except Exception as e:
                stop_flag.set()
                message.reply("❌ Error in agent "+str(e))
            finally:
                self.agent.mode = None
                stop_flag.set()
        threading.Thread(target=helper, daemon=True).start()

    def save_reply(self, message, content):
        res = message.reply(content)
        res_id = res.get("messages", [{}])[0].get("id")
        if res_id is not None:
            self.history.set(res_id, content)
        else:
            print(f"No ID to add to history: {res}")
agent_task(message) async

Initiate email search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
757
758
759
760
761
762
763
764
765
766
767
async def agent_task(self, message):
    """Initiate email search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'self-agent',
        'step': 'await_query'
    }
    return {
        'type': 'quick_reply',
        'text': "Now prompt the self-agent 📝",
        'options': {'cancel': '❌ Cancel Search'}
    }
check_emails(message, query='') async

Improved email checking with WhatsApp API formatting

Source code in toolboxv2/mods/WhatsAppTb/client.py
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
async def check_emails(self, message, query=""):
    """Improved email checking with WhatsApp API formatting"""
    if not self.gmail_service:
        return "⚠️ Gmail service not configured"

    try:
        results = self.gmail_service.users().messages().list(
            userId='me',
            maxResults=10,
            labelIds=['INBOX'],
            q=query
        ).execute()

        emails = []
        for msg in results.get('messages', [])[:10]:
            email_data = self.gmail_service.users().messages().get(
                userId='me',
                id=msg['id'],
                format='metadata'
            ).execute()

            headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
            emails.append({
                'id': msg['id'],
                'from': headers.get('From', 'Unknown'),
                'subject': headers.get('Subject', 'No Subject'),
                'date': headers.get('Date', 'Unknown'),
                'snippet': email_data.get('snippet', ''),
                'unread': 'UNREAD' in email_data.get('labelIds', [])
            })

        return {
            'type': 'list',
            'header': '📨 Recent Emails',
            'body': 'Tap to view full email',
            'footer': 'Email Manager',
            'sections': [{
                'title': f"Inbox ({len(emails)} emails)",
                'rows': [{
                    'id': f"email_{email['id']}",
                    'title': f"{'📬' if email['unread'] else '📭'} {email['subject']}"[:23],
                    'description': f"From: {email['from']}\n{email['snippet']}"[:45]
                } for email in emails]
            }]
        }
    except Exception as e:
        return f"⚠️ Error fetching emails: {str(e)}"
complete_authorization(message)

Complete the authorization process using the authorization code

:param authorization_code: Authorization code received from Google

Source code in toolboxv2/mods/WhatsAppTb/client.py
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
def complete_authorization(self, message: Message):
    """
    Complete the authorization process using the authorization code

    :param authorization_code: Authorization code received from Google
    """
    from google_auth_oauthlib.flow import Flow
    authorization_code = message.content
    # Define the scopes required for Gmail and Calendar
    SCOPES = [
        'https://www.googleapis.com/auth/gmail.modify',
        'https://www.googleapis.com/auth/calendar'
    ]

    # Create a flow instance to manage the OAuth 2.0 authorization process
    flow = Flow.from_client_secrets_file(
        self.credentials_path,
        scopes=SCOPES,
        redirect_uri='urn:ietf:wg:oauth:2.0:oob'
    )

    # Exchange the authorization code for credentials
    flow.fetch_token(code=authorization_code)
    self.credentials = flow.credentials

    # Save the credentials for future use
    self.save_credentials()

    # Initialize services
    self.init_services()
    return "Done"
delete_document(message) async

Delete document workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
1426
1427
1428
1429
1430
1431
1432
1433
1434
async def delete_document(self, message):
    """Delete document workflow"""
    docs = self.blob_docs_system.list_documents()
    return {
        'type': 'quick_reply',
        'text': 'Select document to delete:',
        'options': {doc['id']: doc['name'] for doc in docs[:5]},
        'handler': self._confirm_delete
    }

Initiate email search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
876
877
878
879
880
881
882
883
884
885
886
async def email_search(self, message):
    """Initiate email search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'email_search',
        'step': 'await_query'
    }
    return {
        'type': 'quick_reply',
        'text': "🔍 What would you like to search for?",
        'options': {'cancel': '❌ Cancel Search'}
    }
email_summary(message) async

Generate AI-powered email summaries

Source code in toolboxv2/mods/WhatsAppTb/client.py
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
async def email_summary(self, message):
    """Generate AI-powered email summaries"""
    try:
        messages = self.gmail_service.users().messages().list(
            userId='me',
            maxResults=3,
            labelIds=['INBOX']
        ).execute().get('messages', [])

        email_contents = []
        for msg in messages[:3]:
            email_data = self.gmail_service.users().messages().get(
                userId='me',
                id=msg['id'],
                format='full'
            ).execute()
            email_contents.append(self._parse_email_content(email_data))

        summary = self.agent.mini_task(
            "\n\n".join(email_contents) , "system", "Summarize these emails in bullet points with key details:"
        )

        return f"📋 Email Summary:\n{summary}\n\n*Powered by AI*"
    except Exception as e:
        logging.error(f"Summary failed: {str(e)}")
        return f"❌ Could not generate summary: {str(e)}"
find_time_slot(message) async

Find and display the next 5 available time slots with dynamic durations

Source code in toolboxv2/mods/WhatsAppTb/client.py
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
async def find_time_slot(self, message):
    """Find and display the next 5 available time slots with dynamic durations"""
    if not self.calendar_service:
        return "⚠️ Calendar service not configured"

    try:
        # Define the time range for the search (next 24 hours)
        now = datetime.now(UTC)
        end_time = now + timedelta(days=1)

        # FreeBusy Request
        freebusy_request = {
            "timeMin": now.isoformat(),
            "timeMax": end_time.isoformat(),
            "items": [{"id": 'primary'}]
        }

        freebusy_response = self.calendar_service.freebusy().query(body=freebusy_request).execute()
        busy_slots = freebusy_response['calendars']['primary']['busy']

        # Slot-Berechnung
        available_slots = self._calculate_efficient_slots(
            busy_slots,
            self.duration_minutes
        )

        # Format the response for WhatsApp
        return {
            'type': 'list',
            'header': "⏰ Available Time Slots",
            'body': "Tap to select a time slot",
            'footer': "Time Slot Finder",
            'sections': [{
                'title': "Next 5 Available Slots",
                'rows': [{
                    'id': f"slot_{slot['start'].timestamp()}",
                    'title': f"🕒 {slot['start'].strftime('%H:%M')} - {slot['end'].strftime('%H:%M')}",
                    'description': f"Duration: {slot['duration']}"
                } for slot in available_slots[:5]]
            }]
        }
    except Exception as e:
        return f"⚠️ Error finding time slots: {str(e)}"
generate_authorization_url(*a) async

Generate an authorization URL for user consent

:return: Authorization URL for the user to click and authorize access

Source code in toolboxv2/mods/WhatsAppTb/client.py
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
async def generate_authorization_url(self, *a):
    """
    Generate an authorization URL for user consent

    :return: Authorization URL for the user to click and authorize access
    """
    from google_auth_oauthlib.flow import Flow
    # Define the scopes required for Gmail and Calendar
    SCOPES = [
        'https://www.googleapis.com/auth/gmail.modify',
        'https://www.googleapis.com/auth/calendar'
    ]

    # Create a flow instance to manage the OAuth 2.0 authorization process
    flow = Flow.from_client_secrets_file(
        self.credentials_path,
        scopes=SCOPES,
        redirect_uri='urn:ietf:wg:oauth:2.0:oob'  # Use 'urn:ietf:wg:oauth:2.0:oob' for desktop apps
    )

    # Generate the authorization URL
    authorization_url, _ = flow.authorization_url(
        access_type='offline',  # Allows obtaining refresh token
        prompt='consent'  # Ensures user is always prompted for consent
    )
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'auth',
                                                                          'step': 'awaiting_key'}
    return {
        'type': 'quick_reply',
        'text': f'Url to log in {authorization_url}',
        'options': {'cancel': '❌ Cancel Upload'}
    }
get_email_details(email_id) async

Retrieve and format full email details

Source code in toolboxv2/mods/WhatsAppTb/client.py
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
async def get_email_details(self, email_id):
    """Retrieve and format full email details"""
    if not self.gmail_service:
        return "⚠️ Gmail service not configured"

    try:
        email_data = self.gmail_service.users().messages().get(
            userId='me',
            id=email_id,
            format='full'
        ).execute()

        headers = {h['name']: h['value'] for h in email_data['payload']['headers']}
        body = ""
        for part in email_data.get('payload', {}).get('parts', []):
            if part['mimeType'] == 'text/plain':
                body = base64.urlsafe_b64decode(part['body']['data']).decode('utf-8')
                break

        formatted_text = (
            f"📧 *Email Details*\n\n"
            f"From: {headers.get('From', 'Unknown')}\n"
            f"Subject: {headers.get('Subject', 'No Subject')}\n"
            f"Date: {headers.get('Date', 'Unknown')}\n\n"
            f"{body[:15000]}{'...' if len(body) > 15000 else ''}"
        )
        return  self.agent.mini_task(
            formatted_text , "system", "Summarize the email in bullet points with key details"
        )
    except Exception as e:
        return f"⚠️ Error fetching email: {str(e)}"
get_event_details(event_id) async

Retrieve and format calendar event details with location support

Source code in toolboxv2/mods/WhatsAppTb/client.py
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
async def get_event_details(self, event_id):
    """Retrieve and format calendar event details with location support"""
    if not self.calendar_service:
        return "⚠️ Calendar service not configured"

    try:
        event = self.calendar_service.events().get(
            calendarId='primary',
            eventId=event_id
        ).execute()

        response = [ (
                f"📅 *Event Details*\n\n"
                f"Title: {event.get('summary', 'No title')}\n"
                f"Time: {self._format_event_time(event)}\n"
                f"Location: {event.get('location', 'Not specified')}\n\n"
                f"{event.get('description', 'No description')[:1000]}"
            )]

        if 'geo' in event:
            response.append({
                'lat': float(event['geo']['latitude']),
                'long': float(event['geo']['longitude']),
                'name': event.get('location', 'Event Location'),
                'address': event.get('location', ''),
                'recipient_id': self.whc.progress_messenger0.recipient_phone
            })
        return response
    except Exception as e:
        return f"⚠️ Error fetching event: {str(e)}"
handle_audio_message(message) async

Process audio messages with STT and TTS

Source code in toolboxv2/mods/WhatsAppTb/client.py
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
async def handle_audio_message(self, message: 'Message'):
    """Process audio messages with STT and TTS"""
    # Download audio
    progress = self.progress_messengers['task']
    stop_flag = threading.Event()
    # message_id = progress.send_initial_message(mode="loading")
    progress.message_id = message.id
    progress.start_loading_in_background(stop_flag)

    content = self.whc.messenger.get_audio(message.data)
    audio_file_name = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')), mime_type='audio/opus', file_path=".data/temp")
    print(f"audio_file_name {audio_file_name}")
    if audio_file_name is None:
        message.reply("Could not process audio file")
        stop_flag.set()
        return

    text = self.stt(audio_file_name)['text']
    if not text:
        message.reply("Could not process audio")
        stop_flag.set()
        return

    message.reply("Transcription :\n "+ text)
    message.content = text
    agent_res = await self.helper_text(message, return_text=True)

    if agent_res is not None:
        pass

    stop_flag.set()
handle_button_interaction(content, message) async

Handle button click interactions

Source code in toolboxv2/mods/WhatsAppTb/client.py
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
async def handle_button_interaction(self, content: dict, message: Message):
    """Handle button click interactions"""
    button_id = content['id']

    # First check if it's a main menu button
    if button_id in self.buttons:
        self.whc.messenger.send_button(
            recipient_id=self.whc.progress_messenger0.recipient_phone,
            button=self.buttons[button_id]
        )
        return

    # Handle action buttons
    action_handlers = {
        # Agent controls
        'start': self.start_agent,
        'stop': self.stop_agent,
        'tasks': self.show_task_stack,
        'memory': self.clear_memory,
        'system-task': self.system_task,
        'agent-task': self.agent_task,

        # Email controls
        'check': self.check_emails,
        'send': self.start_email_compose,
        'summary': self.email_summary,
        'search': self.email_search,

        # Calendar controls
        'today': self.show_today_events,
        'add': self.start_event_create,
        'upcoming': self.show_upcoming_events,
        'find_slot': self.find_time_slot,

        # Document controls
        'upload': self.start_document_upload,
        'list': self.list_documents,
        'search_docs': self.search_documents,
        'delete': self.delete_document,

        # System controls
        'status': self.system_status,
        'restart': self.restart_system,
        'connect': self.generate_authorization_url,

        'cancel': self.cancel,
        'confirm': self.confirm,
    }
    if button_id in action_handlers:
        try:
            # Start progress indicator
            progress = self.progress_messengers['task']
            stop_flag = threading.Event()
            # message_id = progress.send_initial_message(mode="loading")
            progress.message_id = message.id
            progress.start_loading_in_background(stop_flag)

            # Execute handler

            result = await action_handlers[button_id](message)


            # Send result
            if isinstance(result, str):
                self.save_reply(message, result)
            elif isinstance(result, dict):  # For structured responses
                self.send_structured_response(result)

            stop_flag.set()
        finally:
            #except Exception as e:
            stop_flag.set()
        #    message.reply(f"❌ Error processing {button_id}: {str(e)}")
    elif 'event_' in button_id:
        res = await self.get_event_details(button_id.replace("event_", ''))
        if isinstance(res, str):
            self.save_reply(message, res)
            return
        for r in res:
            if isinstance(r, str):
                self.save_reply(message, r)
            else:
                self.whc.messenger.send_location(**r)

    elif 'email_' in button_id:
        res = await self.get_email_details(button_id.replace("email_", ''))
        self.save_reply(message, res)
    else:
        message.reply("⚠️ Unknown command")
handle_calendar_actions(message) async

Handle calendar-related pending actions

Source code in toolboxv2/mods/WhatsAppTb/client.py
1193
1194
1195
1196
1197
1198
1199
1200
async def handle_calendar_actions(self, message):
    """Handle calendar-related pending actions"""
    user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

    if user_state.get('type') == 'create_event':
        return await self._handle_event_creation(message, user_state)

    return None
handle_email_actions(message) async

Handle multi-step email workflows

Source code in toolboxv2/mods/WhatsAppTb/client.py
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
    async def handle_email_actions(self, message):
        """Handle multi-step email workflows"""
        user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

        if user_state.get('type') == 'compose_email':
            return await self._handle_email_composition(message, user_state)
        if user_state.get('type') == 'email_search':
            return await self.check_emails(message, self.agent.mini_task("""Conventire Pezise zu einer googel str only query using : Gmail Suchoperatoren!

Basis-Operatoren:
- from: Absender
- to: Empfänger
- subject: Betreff
- label: Gmail Label
- has:attachment Anhänge
- newer_than:7d Zeitfilter
- before: Datum vor
- after: Datum nach

Erweiterte Operatoren:
- in:inbox
- in:sent
- in:spam
- cc: Kopie
- bcc: Blindkopie
- is:unread
- is:read
- larger:10M Größenfilter
- smaller:5M
- filename:pdf Dateityp

Profi-Tipps:
- Kombinierbar mit UND/ODER
- Anführungszeichen für exakte Suche
- Negation mit -
 beispeile : 'Ungelesene Mails letzte Woche': -> 'is:unread newer_than:7d'

""", "user",message.content))


        return None
handle_interactive(message) async

Handle all interactive messages

Source code in toolboxv2/mods/WhatsAppTb/client.py
533
534
535
536
537
538
539
async def handle_interactive(self, message: Message):
    """Handle all interactive messages"""
    content = self.whc.messenger.get_interactive_response(message.data)
    if content.get("type") == "list_reply":
        await self.handle_button_interaction(content.get("list_reply"), message)
    elif content.get("type") == "button_reply":
        print(content)
handle_media_message(message) async

Handle document/image/video uploads

Source code in toolboxv2/mods/WhatsAppTb/client.py
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
async def handle_media_message(self, message: 'Message'):
    """Handle document/image/video uploads"""
    user_state = self.pending_actions.get(self.whc.progress_messenger0.recipient_phone, {})

    if user_state.get('step') == 'awaiting_file':
        file_type = message.type
        if file_type not in ['document', 'image', 'video']:
            return "Unsupported file type"

        try:
            # Download media
            #media_url = message.document.url if hasattr(message, 'document') else \
            #    message.image.url if hasattr(message, 'image') else \
            #        message.video.url
            if file_type =='video':
                content = self.whc.messenger.get_video(message.data)
            if file_type =='image':
                content = self.whc.messenger.get_image(message.data)
            if file_type =='document':
                content = self.whc.messenger.get_document(message.data)
            print("Media content:", content)
            media_data = self.whc.messenger.download_media(media_url=self.whc.messenger.query_media_url(media_id=content.get('id')),  mime_type=content.get('mime_type'), file_path='.data/temp')
            print("Media media_data:", media_data)
            # Save to blob storage
            filename = f"file_{file_type}_{datetime.now().isoformat()}_{content.get('sha256', '')}"
            blob_id = self.blob_docs_system.save_document(
                open(media_data, 'rb').read(),
                filename=filename,
                file_type=file_type
            )

            self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
            return f"✅ File uploaded successfully!\nID: {blob_id}"

        except Exception as e:
            logging.error(f"Upload failed: {str(e)}")
            return f"❌ Failed to upload file Error : {str(e)}"

    return "No pending uploads"
handle_message(message) async

Main message handler for incoming WhatsApp messages

Source code in toolboxv2/mods/WhatsAppTb/client.py
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
async def handle_message(self, message: 'Message'):
    """Main message handler for incoming WhatsApp messages"""

    # Deduplication check
    with self.message_lock:
        if message.id in self.processed_messages:
            return
        last_ts = time.time()
        print(last_ts)
        if len(self.processed_messages) > 0:
            m_id, last_ts = self.processed_messages.pop()
            self.processed_messages.add((m_id, last_ts))

        print("DUPLICATION P", message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0) , last_ts)
        if float(message.data.get('entry', [{}])[0].get('changes', [{}])[0].get('value', {}).get('messages', [{}])[0].get('timestamp', 0)) < last_ts - 120:
            return
        self.processed_messages.add((message.id, time.perf_counter()))

    # Mark message as read
    message.mark_as_read()

    # Extract content and type
    content_type = message.type
    content = message.content

    print(f"message.content {content=} {content_type=} {message.data=}")

    try:
        if content_type == 'interactive':
            await self.handle_interactive(message)
        elif content_type == 'audio':
            await self.handle_audio_message(message)
        elif content_type in ['document', 'image', 'video']:
            response = await self.handle_media_message(message)
            self.save_reply(message, response)
        elif content_type == 'text':
            if content.lower() == "menu":
                self.whc.messenger.send_button(
                    recipient_id=self.whc.progress_messenger0.recipient_phone,
                    button=self.buttons[content.lower()]
                )
            else:
                await self.helper_text(message)
        else:
            message.reply("Unsupported message type")
    #except Exception as e:
    #    logging.error(f"Message handling error: {str(e)}")
    #   message.reply("❌ Error processing request")
    finally:
        # Cleanup old messages (keep 1 hour history)
        with self.message_lock:
            self._clean_processed_messages()
init_services()

Initialize Gmail and Calendar services

Source code in toolboxv2/mods/WhatsAppTb/client.py
281
282
283
284
285
286
287
288
289
def init_services(self):
    """
    Initialize Gmail and Calendar services
    """
    from googleapiclient.discovery import build

    self.gmail_service = build('gmail', 'v1', credentials=self.credentials)
    self.calendar_service = build('calendar', 'v3', credentials=self.credentials)
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {}
load_credentials()

Load previously saved credentials if available

:return: Whether credentials were successfully loaded

Source code in toolboxv2/mods/WhatsAppTb/client.py
267
268
269
270
271
272
273
274
275
276
277
278
def load_credentials(self):
    """
    Load previously saved credentials if available

    :return: Whether credentials were successfully loaded
    """
    try:
        self.credentials = Credentials.from_authorized_user_file('token/google_token.json')
        self.init_services()
        return True
    except FileNotFoundError:
        return False
run()

Start the WhatsApp assistant

Source code in toolboxv2/mods/WhatsAppTb/client.py
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
def run(self):
    """Start the WhatsApp assistant"""
    try:
        self.state = AssistantState.ONLINE
        # Send welcome message

        mas = self.whc.messenger.create_message(
            content="Digital Assistant is online! Send /help for available commands.",to=self.whc.progress_messenger0.recipient_phone,
        ).send(sender=0)
        mas_id = mas.get("messages", [{}])[0].get("id")
        print(mas_id)

    except Exception as e:
        logging.error(f"Assistant error: {str(e)}")
        self.state = AssistantState.OFFLINE
        raise
save_credentials()

Save the obtained credentials to a file for future use

Source code in toolboxv2/mods/WhatsAppTb/client.py
256
257
258
259
260
261
262
263
264
def save_credentials(self):
    """
    Save the obtained credentials to a file for future use
    """
    if not os.path.exists('token'):
        os.makedirs('token')

    with open('token/google_token.json', 'w') as token_file:
        token_file.write(self.credentials.to_json())
search_documents(message) async

Initiate document search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
1377
1378
1379
1380
1381
1382
1383
1384
async def search_documents(self, message):
    """Initiate document search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'search', 'step': 'awaiting_query'}
    return {
        'type': 'quick_reply',
        'text': '🔍 What are you looking for?',
        'options': {'cancel': '❌ Cancel Search'}
    }
send_email(to, subject, body)

Actual email sending function to be called by agent

Source code in toolboxv2/mods/WhatsAppTb/client.py
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
def send_email(self, to, subject, body):
    """Actual email sending function to be called by agent"""
    if not self.gmail_service:
        return False

    message = MIMEText(body)
    message['to'] = to
    message['subject'] = subject

    encoded_message = base64.urlsafe_b64encode(message.as_bytes()).decode()
    self.gmail_service.users().messages().send(
        userId='me',
        body={'raw': encoded_message}
    ).execute()
    return True
send_structured_response(result)

Send complex responses using appropriate WhatsApp features

Source code in toolboxv2/mods/WhatsAppTb/client.py
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
def send_structured_response(self, result: dict):
    """Send complex responses using appropriate WhatsApp features"""
    if result['type'] == 'list':
        self.whc.messenger.send_button(
            recipient_id=self.whc.progress_messenger0.recipient_phone,
            button={
                'header': result.get('header', ''),
                'body': result.get('body', ''),
                'footer': result.get('footer', ''),
                'action': {
                    'button': 'Action',
                    'sections': result['sections']
                }
            }
        )
    elif result['type'] == 'quick_reply':
        self.whc.messenger.send_button(
            recipient_id=self.whc.progress_messenger0.recipient_phone,
            button={
                'header': "Quick reply",
                'body': result['text'],
                'footer': '',
                'action': {'button': 'Action', 'sections': [{
                    'title': 'View',
                    'rows': [{'id': k, 'title': v[:23]} for k, v in result['options'].items()]
                }]}
            }
        )

    elif result['type'] == 'media':
        if result['media_type'] == 'image':
            self.whc.messenger.send_image(
                image=result['url'],
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                caption=result.get('caption', '')
            )
        elif result['media_type'] == 'document':
            self.whc.messenger.send_document(
                document=result['url'],
                recipient_id=self.whc.progress_messenger0.recipient_phone,
                caption=result.get('caption', '')
            )
setup_interaction_buttons()

Define WhatsApp interaction buttons for different functionalities

Source code in toolboxv2/mods/WhatsAppTb/client.py
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
def setup_interaction_buttons(self):
    """Define WhatsApp interaction buttons for different functionalities"""
    self.buttons = {
        'menu': {
            'header': 'Digital Assistant',
            'body': 'Please select an option:',
            'footer': '-- + --',
            'action': {
                'button': 'Menu',
                'sections': [
                    {
                        'title': 'Main Functions',
                        'rows': [
                            {'id': 'agent', 'title': 'Agent Controls', 'description': 'Manage your AI assistant'},
                            {'id': 'email', 'title': 'Email Management', 'description': 'Handle your emails'},
                            {'id': 'calendar', 'title': 'Calendar', 'description': 'Manage your schedule'},
                            {'id': 'docs', 'title': 'Documents', 'description': 'Handle documents'},
                            {'id': 'system', 'title': 'System', 'description': 'System controls and metrics'}
                        ]
                    }
                ]
            }
        },
        'agent': self._create_agent_controls_buttons(),
        'email': self._create_email_controls_buttons(),
        'calendar': self._create_calendar_controls_buttons(),
        'docs': self._create_docs_controls_buttons(),
        'system': self._create_system_controls_buttons()
    }
setup_progress_messengers()

Initialize progress messengers for different types of tasks

Source code in toolboxv2/mods/WhatsAppTb/client.py
291
292
293
294
295
296
297
def setup_progress_messengers(self):
    """Initialize progress messengers for different types of tasks"""
    self.progress_messengers = {
        'task': self.whc.progress_messenger0,
        'email': self.whc.progress_messenger1,
        'calendar': self.whc.progress_messenger2
    }
show_task_stack(*a) async

Display current task stack

Source code in toolboxv2/mods/WhatsAppTb/client.py
1504
1505
1506
1507
1508
1509
async def show_task_stack(self, *a):
    """Display current task stack"""
    if self.agent and len(self.agent.taskstack.tasks) > 0:
        tasks = self.agent.taskstack.tasks
        return self.agent.mini_task("\n".join([f"Task {t.id}: {t.description}" for t in tasks]), "system", "Format to nice and clean whatsapp format")
    return "No tasks in stack"
show_today_events(message) async

Show today's calendar events

Source code in toolboxv2/mods/WhatsAppTb/client.py
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
async def show_today_events(self, message):
    """Show today's calendar events"""
    if not self.calendar_service:
        message.replay("service not online")

    now = datetime.utcnow().isoformat() + 'Z'
    end_of_day = (datetime.now() + timedelta(days=1)).replace(
        hour=0, minute=0, second=0).isoformat() + 'Z'

    events_result = self.calendar_service.events().list(
        calendarId='primary',
        timeMin=now,
        timeMax=end_of_day,
        singleEvents=True,
        orderBy='startTime'
    ).execute()

    events = events_result.get('items', [])
    return self._format_calendar_response(events, "Today's Events")
show_upcoming_events(message) async

Show upcoming events with interactive support

Source code in toolboxv2/mods/WhatsAppTb/client.py
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
async def show_upcoming_events(self, message):
    """Show upcoming events with interactive support"""
    if not self.calendar_service:
        return "⚠️ Calendar service not configured"

    try:
        now = datetime.utcnow().isoformat() + 'Z'
        next_week = (datetime.now() + timedelta(days=7)).isoformat() + 'Z'

        events_result = self.calendar_service.events().list(
            calendarId='primary',
            timeMin=now,
            timeMax=next_week,
            singleEvents=True,
            orderBy='startTime',
            maxResults=10
        ).execute()

        events = events_result.get('items', [])
        return self._format_calendar_response(events, "Upcoming Events")
    except Exception as e:
        return f"⚠️ Error fetching events: {str(e)}"
start_agent(*a) async

Start the agent in background mode

Source code in toolboxv2/mods/WhatsAppTb/client.py
1490
1491
1492
1493
1494
1495
async def start_agent(self, *a):
    """Start the agent in background mode"""
    if self.agent:
        self.agent.run_in_background()
        return True
    return False
start_document_upload(message) async

Initiate document upload workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
1368
1369
1370
1371
1372
1373
1374
1375
async def start_document_upload(self, message):
    """Initiate document upload workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {'type': 'document', 'step': 'awaiting_file'}
    return {
        'type': 'quick_reply',
        'text': '📤 Send me the file you want to upload',
        'options': {'cancel': '❌ Cancel Upload'}
    }
start_email_compose(message) async

Enhanced email composition workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
888
889
890
891
892
893
894
895
896
897
898
899
async def start_email_compose(self, message):
    """Enhanced email composition workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'compose_email',
        'step': 'subject',
        'draft': {'attachments': []}
    }
    return {
        'type': 'quick_reply',
        'text': "📝 Let's compose an email\n\nSubject:",
        'options': {'cancel': '❌ Cancel Composition'}
    }
start_event_create(message) async

Initiate event creation workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
async def start_event_create(self, message):
    """Initiate event creation workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'create_event',
        'step': 'title',
        'event_data': {}
    }
    return {
        'type': 'quick_reply',
        'text': "Let's create an event! What's the title?",
        'options': {'cancel': '❌ Cancel'}
    }
stop_agent(*b) async

Stop the currently running agent

Source code in toolboxv2/mods/WhatsAppTb/client.py
1497
1498
1499
1500
1501
1502
async def stop_agent(self, *b):
    """Stop the currently running agent"""
    if self.agent:
        self.agent.stop()
        return True
    return False
system_task(message) async

Initiate email search workflow

Source code in toolboxv2/mods/WhatsAppTb/client.py
745
746
747
748
749
750
751
752
753
754
755
async def system_task(self, message):
    """Initiate email search workflow"""
    self.pending_actions[self.whc.progress_messenger0.recipient_phone] = {
        'type': 'system',
        'step': 'await_query'
    }
    return {
        'type': 'quick_reply',
        'text': "Now prompt the 🧠ISAA-System 📝",
        'options': {'cancel': '❌ Cancel Search'}
    }

server

AppManager
Source code in toolboxv2/mods/WhatsAppTb/server.py
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
class AppManager(metaclass=Singleton):
    pepper = "pepper0"

    def __init__(self, start_port: int = 8000, port_range: int = 10, em=None):
        self.instances: dict[str, dict] = {}
        self.start_port = start_port
        self.port_range = port_range
        self.threads: dict[str, Thread] = {}
        self.stop_events: dict[str, Event] = {}
        self.message_queue: asyncio.Queue = asyncio.Queue()
        self.last_messages: dict[str, datetime] = {}
        self.keys: dict[str, str] = {}
        self.forwarders: dict[str, dict] = {}
        self.runner = lambda :None

        if em is None:
            from toolboxv2 import get_app
            em = get_app().get_mod("EventManager")
        from toolboxv2.mods import EventManager
        self.event_manager: EventManager = em.get_manager()

        # Set up signal handlers for graceful shutdown
        try:
            if threading.current_thread() is threading.main_thread():
                signal.signal(signal.SIGINT, self.signal_handler)
                signal.signal(signal.SIGTERM, self.signal_handler)
        except Exception:
            pass

    def offline(self, instance_id):

        def mark_as_offline():
            self.forwarders[instance_id]['send'] = None
            return 'done'

        return mark_as_offline

    def online(self, instance_id):

        def mark_as_online():
            return self.instances[instance_id]['app']

        def set_callbacks(callback, e_callback=None):
            if callback is not None:
                self.forwarders[instance_id]['send'] = callback
            if e_callback is not None:
                self.forwarders[instance_id]['sende'] = e_callback

        return mark_as_online(), set_callbacks

    def get_next_available_port(self) -> int:
        """Find the next available port in the range."""
        used_ports = {instance['port'] for instance in self.instances.values()}
        for port in range(self.start_port, self.start_port + self.port_range):
            if port not in used_ports:
                return port
        raise RuntimeError("No available ports in range")

    def add_instance(self, instance_id: str, **kwargs):
        """
        Add a new app instance to the manager with automatic port assignment.
        """
        if instance_id in self.instances:
            raise ValueError(f"Instance {instance_id} already exists")

        port = self.get_next_available_port()
        app_instance = WhatsApp(**kwargs)

        self.instances[instance_id] = {
            'app': app_instance,
            'port': port,
            'kwargs': kwargs,
            'phone_number_id': kwargs.get("phone_number_id", {}),
            'retry_count': 0,
            'max_retries': 3,
            'retry_delay': 5
        }
        self.keys[instance_id] = Code.one_way_hash(kwargs.get("phone_number_id", {}).get("key"), "WhatsappAppManager",
                                                   self.pepper)
        self.forwarders[instance_id] = {}

        # Set up message handlers
        @app_instance.on_message
        async def message_handler(message):
            await self.on_message(instance_id, message)

        @app_instance.on_event
        async def event_handler(event):
            await self.on_event(instance_id, event)

        @app_instance.on_verification
        async def verification_handler(verification):
            await self.on_verification(instance_id, verification)

        # Create stop event for this instance Error parsing message1:
        self.stop_events[instance_id] = Event()

    def run_instance(self, instance_id: str):
        """Run a single instance in a separate thread with error handling and automatic restart."""
        instance_data = self.instances[instance_id]
        stop_event = self.stop_events[instance_id]

        while not stop_event.is_set():
            try:
                logger.info(f"Starting instance {instance_id} on port {instance_data['port']}")
                instance_data['app'].run(host='0.0.0.0', port=instance_data['port'])

            except Exception as e:
                logger.error(f"Error in instance {instance_id}: {str(e)}")
                instance_data['retry_count'] += 1

                if instance_data['retry_count'] > instance_data['max_retries']:
                    logger.error(f"Max retries exceeded for instance {instance_id}")
                    break

                logger.info(f"Restarting instance {instance_id} in {instance_data['retry_delay']} seconds...")
                time.sleep(instance_data['retry_delay'])

                # Recreate the instance
                instance_data['app'] = WhatsApp(**instance_data['kwargs'])
                continue

    async def on_message(self, instance_id: str, message: Message):
        """Handle and forward incoming messages."""
        logger.info(f"Message from instance {instance_id}: {message}")
        if instance_id in self.forwarders and 'send' in self.forwarders[instance_id]:
            await self.forwarders[instance_id]['send'](message)

    async def on_event(self, instance_id: str, event):
        """Handle events."""
        logger.info(f"Event from instance {instance_id}: {event}")
        if instance_id in self.forwarders and 'sende' in self.forwarders[instance_id] and self.forwarders[instance_id]['sende'] is not None:
            self.forwarders[instance_id]['sende'](event)

    async def on_verification(self, instance_id: str, verification):
        """Handle verification events."""
        logger.info(f"Verification from instance {instance_id}: {verification}")

    def run_all_instances(self):
        """Start all instances in separate daemon threads."""
        # Start message forwarder

        # Start all instances
        for instance_id in self.instances:
            thread = Thread(
                target=self.run_instance,
                args=(instance_id,),
                daemon=True,
                name=f"WhatsApp-{instance_id}"
            )
            self.threads[instance_id] = thread
            thread.start()

    def signal_handler(self, signum, frame):
        """Handle shutdown signals gracefully."""
        logger.info("Shutdown signal received, stopping all instances...")
        self.stop_all_instances()
        sys.exit(0)

    def stop_all_instances(self):
        """Stop all running instances gracefully."""
        for instance_id in self.stop_events:
            self.stop_events[instance_id].set()

        for thread in self.threads.values():
            thread.join(timeout=5)

    def create_manager_ui(self, start_assistant):
        """Enhanced WhatsApp Manager UI with instance configuration controls"""
        self.runner = start_assistant
        def ui_manager():
            # Track instance states and messages
            original_on_message = self.on_message

            async def enhanced_on_message(instance_id: str, message):
                self.last_messages[instance_id] = datetime.now()
                await original_on_message(instance_id, message)

            self.on_message = enhanced_on_message

            def create_instance_card(instance_id: str):
                """Interactive instance control card"""
                config = self.instances[instance_id]
                with ui.card().classes('w-full p-4 mb-4 bg-gray-50 dark:bg-gray-800').style("background-color: var(--background-color) !important"):
                    # Header Section
                    with ui.row().classes('w-full justify-between items-center'):
                        ui.label(f'📱 {instance_id}').classes('text-xl font-bold')

                        # Status Indicator
                        ui.label().bind_text_from(
                            self.threads, instance_id,
                            lambda x: 'Running' if x and x.is_alive() else 'Stopped'
                        )

                    # Configuration Display
                    with ui.grid(columns=2).classes('w-full mt-4 gap-2'):

                        ui.label('port:').classes('font-bold')
                        ui.label(config['port'])

                        ui.label('Last Activity:').classes('font-bold')
                        ui.label().bind_text_from(
                            self.last_messages, instance_id,
                            lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if x else 'Never'
                        )

                    # Action Controls
                    with ui.row().classes('w-full mt-4 gap-2'):
                        with ui.button(icon='settings', on_click=lambda: edit_dialog.open()).props('flat'):
                            ui.tooltip('Configure')

                        with ui.button(icon='refresh', color='orange',
                                       on_click=lambda: self.restart_instance(instance_id)):
                            ui.tooltip('Restart')

                        with ui.button(icon='stop', color='red',
                                       on_click=lambda: self.stop_instance(instance_id)):
                            ui.tooltip('Stop')

                    # Edit Configuration Dialog
                    with ui.dialog() as edit_dialog, ui.card().classes('p-4 gap-4'):
                        new_key = ui.input('API Key', value=config['phone_number_id'].get('key', ''))
                        new_number = ui.input('Phone Number', value=config['phone_number_id'].get('number', ''))

                        with ui.row().classes('w-full justify-end'):
                            ui.button('Cancel', on_click=edit_dialog.close)
                            ui.button('Save', color='primary', on_click=lambda: (
                                self.update_instance_config(
                                    instance_id,
                                    new_key.value,
                                    new_number.value
                                ),
                                edit_dialog.close()
                            ))

            # Main UI Layout
            with ui.column().classes('w-full max-w-4xl mx-auto p-4'):
                ui.label('WhatsApp Instance Manager').classes('text-2xl font-bold mb-6')

                # Add Instance Section
                with ui.expansion('➕ Add New Instance', icon='add').classes('w-full'):
                    with ui.card().classes('w-full p-4 mt-2'):
                        instance_id = ui.input('Instance ID').classes('w-full')
                        token = ui.input('API Token').classes('w-full')
                        phone_key = ui.input('Phone Number Key').classes('w-full')
                        phone_number = ui.input('Phone Number').classes('w-full')

                        with ui.row().classes('w-full justify-end gap-2'):
                            ui.button('Clear', on_click=lambda: (
                                instance_id.set_value(''),
                                token.set_value(''),
                                phone_key.set_value(''),
                                phone_number.set_value('')
                            ))
                            ui.button('Create', color='positive', on_click=lambda: (
                                self.add_update_instance(
                                    instance_id.value,
                                    token.value,
                                    phone_key.value,
                                    phone_number.value
                                ),
                                instances_container.refresh()
                            ))

                # Instances Display
                instances_container = ui.column().classes('w-full')
                with instances_container:
                    for instance_id in self.instances:
                        create_instance_card(instance_id)

        return ui_manager

    # Add to manager class
    def add_update_instance(self, instance_id, token, phone_key, phone_number):
        """Add or update instance configuration"""
        if instance_id in self.instances:
            self.stop_instance(instance_id)
            del self.instances[instance_id]

        self.add_instance(
            instance_id,
            token=token,
            phone_number_id={
                'key': phone_key,
                'number': phone_number
            },
            verify_token=os.getenv("WHATSAPP_VERIFY_TOKEN")
        )
        self.start_instance(instance_id)

    def update_instance_config(self, instance_id, new_key, new_number):
        """Update existing instance configuration"""
        if instance_id in self.instances:
            self.instances[instance_id]['phone_number_id'] = {
                'key': new_key,
                'number': new_number
            }
            self.restart_instance(instance_id)

    def restart_instance(self, instance_id):
        """Safe restart of instance"""
        self.stop_instance(instance_id)
        self.start_instance(instance_id)

    def stop_instance(self, instance_id):
        """Graceful stop of instance"""
        if instance_id in self.threads:
            self.stop_events[instance_id].set()
            self.threads[instance_id].join(timeout=5)
            del self.threads[instance_id]

    def start_instance(self, instance_id):
        """Start instance thread"""
        print("Starting Istance")

        self.stop_events[instance_id] = threading.Event()
        self.threads[instance_id] = threading.Thread(
            target=self.run_instance,
            args=(instance_id,),
            daemon=True
        )
        self.threads[instance_id].start()
        print("Running starter", self.runner())
add_instance(instance_id, **kwargs)

Add a new app instance to the manager with automatic port assignment.

Source code in toolboxv2/mods/WhatsAppTb/server.py
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
def add_instance(self, instance_id: str, **kwargs):
    """
    Add a new app instance to the manager with automatic port assignment.
    """
    if instance_id in self.instances:
        raise ValueError(f"Instance {instance_id} already exists")

    port = self.get_next_available_port()
    app_instance = WhatsApp(**kwargs)

    self.instances[instance_id] = {
        'app': app_instance,
        'port': port,
        'kwargs': kwargs,
        'phone_number_id': kwargs.get("phone_number_id", {}),
        'retry_count': 0,
        'max_retries': 3,
        'retry_delay': 5
    }
    self.keys[instance_id] = Code.one_way_hash(kwargs.get("phone_number_id", {}).get("key"), "WhatsappAppManager",
                                               self.pepper)
    self.forwarders[instance_id] = {}

    # Set up message handlers
    @app_instance.on_message
    async def message_handler(message):
        await self.on_message(instance_id, message)

    @app_instance.on_event
    async def event_handler(event):
        await self.on_event(instance_id, event)

    @app_instance.on_verification
    async def verification_handler(verification):
        await self.on_verification(instance_id, verification)

    # Create stop event for this instance Error parsing message1:
    self.stop_events[instance_id] = Event()
add_update_instance(instance_id, token, phone_key, phone_number)

Add or update instance configuration

Source code in toolboxv2/mods/WhatsAppTb/server.py
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
def add_update_instance(self, instance_id, token, phone_key, phone_number):
    """Add or update instance configuration"""
    if instance_id in self.instances:
        self.stop_instance(instance_id)
        del self.instances[instance_id]

    self.add_instance(
        instance_id,
        token=token,
        phone_number_id={
            'key': phone_key,
            'number': phone_number
        },
        verify_token=os.getenv("WHATSAPP_VERIFY_TOKEN")
    )
    self.start_instance(instance_id)
create_manager_ui(start_assistant)

Enhanced WhatsApp Manager UI with instance configuration controls

Source code in toolboxv2/mods/WhatsAppTb/server.py
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
def create_manager_ui(self, start_assistant):
    """Enhanced WhatsApp Manager UI with instance configuration controls"""
    self.runner = start_assistant
    def ui_manager():
        # Track instance states and messages
        original_on_message = self.on_message

        async def enhanced_on_message(instance_id: str, message):
            self.last_messages[instance_id] = datetime.now()
            await original_on_message(instance_id, message)

        self.on_message = enhanced_on_message

        def create_instance_card(instance_id: str):
            """Interactive instance control card"""
            config = self.instances[instance_id]
            with ui.card().classes('w-full p-4 mb-4 bg-gray-50 dark:bg-gray-800').style("background-color: var(--background-color) !important"):
                # Header Section
                with ui.row().classes('w-full justify-between items-center'):
                    ui.label(f'📱 {instance_id}').classes('text-xl font-bold')

                    # Status Indicator
                    ui.label().bind_text_from(
                        self.threads, instance_id,
                        lambda x: 'Running' if x and x.is_alive() else 'Stopped'
                    )

                # Configuration Display
                with ui.grid(columns=2).classes('w-full mt-4 gap-2'):

                    ui.label('port:').classes('font-bold')
                    ui.label(config['port'])

                    ui.label('Last Activity:').classes('font-bold')
                    ui.label().bind_text_from(
                        self.last_messages, instance_id,
                        lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if x else 'Never'
                    )

                # Action Controls
                with ui.row().classes('w-full mt-4 gap-2'):
                    with ui.button(icon='settings', on_click=lambda: edit_dialog.open()).props('flat'):
                        ui.tooltip('Configure')

                    with ui.button(icon='refresh', color='orange',
                                   on_click=lambda: self.restart_instance(instance_id)):
                        ui.tooltip('Restart')

                    with ui.button(icon='stop', color='red',
                                   on_click=lambda: self.stop_instance(instance_id)):
                        ui.tooltip('Stop')

                # Edit Configuration Dialog
                with ui.dialog() as edit_dialog, ui.card().classes('p-4 gap-4'):
                    new_key = ui.input('API Key', value=config['phone_number_id'].get('key', ''))
                    new_number = ui.input('Phone Number', value=config['phone_number_id'].get('number', ''))

                    with ui.row().classes('w-full justify-end'):
                        ui.button('Cancel', on_click=edit_dialog.close)
                        ui.button('Save', color='primary', on_click=lambda: (
                            self.update_instance_config(
                                instance_id,
                                new_key.value,
                                new_number.value
                            ),
                            edit_dialog.close()
                        ))

        # Main UI Layout
        with ui.column().classes('w-full max-w-4xl mx-auto p-4'):
            ui.label('WhatsApp Instance Manager').classes('text-2xl font-bold mb-6')

            # Add Instance Section
            with ui.expansion('➕ Add New Instance', icon='add').classes('w-full'):
                with ui.card().classes('w-full p-4 mt-2'):
                    instance_id = ui.input('Instance ID').classes('w-full')
                    token = ui.input('API Token').classes('w-full')
                    phone_key = ui.input('Phone Number Key').classes('w-full')
                    phone_number = ui.input('Phone Number').classes('w-full')

                    with ui.row().classes('w-full justify-end gap-2'):
                        ui.button('Clear', on_click=lambda: (
                            instance_id.set_value(''),
                            token.set_value(''),
                            phone_key.set_value(''),
                            phone_number.set_value('')
                        ))
                        ui.button('Create', color='positive', on_click=lambda: (
                            self.add_update_instance(
                                instance_id.value,
                                token.value,
                                phone_key.value,
                                phone_number.value
                            ),
                            instances_container.refresh()
                        ))

            # Instances Display
            instances_container = ui.column().classes('w-full')
            with instances_container:
                for instance_id in self.instances:
                    create_instance_card(instance_id)

    return ui_manager
get_next_available_port()

Find the next available port in the range.

Source code in toolboxv2/mods/WhatsAppTb/server.py
78
79
80
81
82
83
84
def get_next_available_port(self) -> int:
    """Find the next available port in the range."""
    used_ports = {instance['port'] for instance in self.instances.values()}
    for port in range(self.start_port, self.start_port + self.port_range):
        if port not in used_ports:
            return port
    raise RuntimeError("No available ports in range")
on_event(instance_id, event) async

Handle events.

Source code in toolboxv2/mods/WhatsAppTb/server.py
156
157
158
159
160
async def on_event(self, instance_id: str, event):
    """Handle events."""
    logger.info(f"Event from instance {instance_id}: {event}")
    if instance_id in self.forwarders and 'sende' in self.forwarders[instance_id] and self.forwarders[instance_id]['sende'] is not None:
        self.forwarders[instance_id]['sende'](event)
on_message(instance_id, message) async

Handle and forward incoming messages.

Source code in toolboxv2/mods/WhatsAppTb/server.py
150
151
152
153
154
async def on_message(self, instance_id: str, message: Message):
    """Handle and forward incoming messages."""
    logger.info(f"Message from instance {instance_id}: {message}")
    if instance_id in self.forwarders and 'send' in self.forwarders[instance_id]:
        await self.forwarders[instance_id]['send'](message)
on_verification(instance_id, verification) async

Handle verification events.

Source code in toolboxv2/mods/WhatsAppTb/server.py
162
163
164
async def on_verification(self, instance_id: str, verification):
    """Handle verification events."""
    logger.info(f"Verification from instance {instance_id}: {verification}")
restart_instance(instance_id)

Safe restart of instance

Source code in toolboxv2/mods/WhatsAppTb/server.py
327
328
329
330
def restart_instance(self, instance_id):
    """Safe restart of instance"""
    self.stop_instance(instance_id)
    self.start_instance(instance_id)
run_all_instances()

Start all instances in separate daemon threads.

Source code in toolboxv2/mods/WhatsAppTb/server.py
166
167
168
169
170
171
172
173
174
175
176
177
178
179
def run_all_instances(self):
    """Start all instances in separate daemon threads."""
    # Start message forwarder

    # Start all instances
    for instance_id in self.instances:
        thread = Thread(
            target=self.run_instance,
            args=(instance_id,),
            daemon=True,
            name=f"WhatsApp-{instance_id}"
        )
        self.threads[instance_id] = thread
        thread.start()
run_instance(instance_id)

Run a single instance in a separate thread with error handling and automatic restart.

Source code in toolboxv2/mods/WhatsAppTb/server.py
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
def run_instance(self, instance_id: str):
    """Run a single instance in a separate thread with error handling and automatic restart."""
    instance_data = self.instances[instance_id]
    stop_event = self.stop_events[instance_id]

    while not stop_event.is_set():
        try:
            logger.info(f"Starting instance {instance_id} on port {instance_data['port']}")
            instance_data['app'].run(host='0.0.0.0', port=instance_data['port'])

        except Exception as e:
            logger.error(f"Error in instance {instance_id}: {str(e)}")
            instance_data['retry_count'] += 1

            if instance_data['retry_count'] > instance_data['max_retries']:
                logger.error(f"Max retries exceeded for instance {instance_id}")
                break

            logger.info(f"Restarting instance {instance_id} in {instance_data['retry_delay']} seconds...")
            time.sleep(instance_data['retry_delay'])

            # Recreate the instance
            instance_data['app'] = WhatsApp(**instance_data['kwargs'])
            continue
signal_handler(signum, frame)

Handle shutdown signals gracefully.

Source code in toolboxv2/mods/WhatsAppTb/server.py
181
182
183
184
185
def signal_handler(self, signum, frame):
    """Handle shutdown signals gracefully."""
    logger.info("Shutdown signal received, stopping all instances...")
    self.stop_all_instances()
    sys.exit(0)
start_instance(instance_id)

Start instance thread

Source code in toolboxv2/mods/WhatsAppTb/server.py
339
340
341
342
343
344
345
346
347
348
349
350
def start_instance(self, instance_id):
    """Start instance thread"""
    print("Starting Istance")

    self.stop_events[instance_id] = threading.Event()
    self.threads[instance_id] = threading.Thread(
        target=self.run_instance,
        args=(instance_id,),
        daemon=True
    )
    self.threads[instance_id].start()
    print("Running starter", self.runner())
stop_all_instances()

Stop all running instances gracefully.

Source code in toolboxv2/mods/WhatsAppTb/server.py
187
188
189
190
191
192
193
def stop_all_instances(self):
    """Stop all running instances gracefully."""
    for instance_id in self.stop_events:
        self.stop_events[instance_id].set()

    for thread in self.threads.values():
        thread.join(timeout=5)
stop_instance(instance_id)

Graceful stop of instance

Source code in toolboxv2/mods/WhatsAppTb/server.py
332
333
334
335
336
337
def stop_instance(self, instance_id):
    """Graceful stop of instance"""
    if instance_id in self.threads:
        self.stop_events[instance_id].set()
        self.threads[instance_id].join(timeout=5)
        del self.threads[instance_id]
update_instance_config(instance_id, new_key, new_number)

Update existing instance configuration

Source code in toolboxv2/mods/WhatsAppTb/server.py
318
319
320
321
322
323
324
325
def update_instance_config(self, instance_id, new_key, new_number):
    """Update existing instance configuration"""
    if instance_id in self.instances:
        self.instances[instance_id]['phone_number_id'] = {
            'key': new_key,
            'number': new_number
        }
        self.restart_instance(instance_id)

utils

ProgressMessenger
Source code in toolboxv2/mods/WhatsAppTb/utils.py
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
class ProgressMessenger:
    def __init__(self, messenger, recipient_phone: str, max_steps: int = 5, emoji_set: list[str] = None, content=None):
        self.messenger = messenger
        self.recipient_phone = recipient_phone
        self.max_steps = max_steps
        self.emoji_set = emoji_set or ["⬜", "⬛", "🟩", "🟨", "🟦"]
        self.message_id = None
        self.content = content

    def send_initial_message(self, mode: str = "progress"):
        """
        Sends the initial message. Modes can be 'progress' or 'loading'.
        """
        if mode == "progress":
            emoji_legend = "\n".join(
                f"{emoji} - Step {i + 1}" for i, emoji in enumerate(self.emoji_set)
            )
            content = (
                "Progress is being updated in real-time!\n\n"
                "Legend:\n"
                f"{emoji_legend}\n\n"
                "Stay tuned for updates!"
            )
        elif mode == "loading":
            content = (
                "Loading in progress! 🌀\n"
                "The indicator will loop until work is done."
            )
        else:
            raise ValueError("Invalid mode. Use 'progress' or 'loading'.")

        if self.content is not None:
            content += '\n'+self.content
        message = self.messenger.create_message(content=content, to=self.recipient_phone)
        response = message.send(sender=0)
        self.message_id = response.get("messages", [{}])[0].get("id")
        logging.info(f"Initial message sent: {content}")
        return self.message_id

    def update_progress(self, step_flag: threading.Event):
        """
        Updates the reaction on the message to represent progress.
        """
        if not self.message_id:
            raise ValueError("Message ID not found. Ensure the initial message is sent first.")
        message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
        for step in range(self.max_steps):
            emoji = self.emoji_set[step % len(self.emoji_set)]
            message.react(emoji)
            logging.info(f"Progress updated: Step {step + 1}/{self.max_steps} with emoji {emoji}")
            while not step_flag.is_set():
                time.sleep(0.5)
            step_flag.clear()
        # Final acknowledgment
        message.react("👍")
        logging.info("Progress completed with final acknowledgment.")

    def update_loading(self, stop_flag: threading.Event):
        """
        Continuously updates the reaction to represent a looping 'loading' indicator.
        """
        if not self.message_id:
            raise ValueError("Message ID not found. Ensure the initial message is sent first.")
        message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
        step = 0
        while not stop_flag.is_set():
            emoji = self.emoji_set[step % len(self.emoji_set)]
            message.react(emoji)
            logging.info(f"Loading update: {emoji}")
            time.sleep(1)  # Faster updates for loading
            step += 1
        # Final acknowledgment
        message.react("✅")
        logging.info("Loading completed with final acknowledgment.")
        message.reply("✅Done✅")

    def start_progress_in_background(self, step_flag):
        """
        Starts the progress update in a separate thread.
        """
        threading.Thread(target=self.update_progress, args=(step_flag, ), daemon=True).start()

    def start_loading_in_background(self, stop_flag: threading.Event):
        """
        Starts the loading update in a separate thread.
        """
        threading.Thread(target=self.update_loading, args=(stop_flag,), daemon=True).start()
send_initial_message(mode='progress')

Sends the initial message. Modes can be 'progress' or 'loading'.

Source code in toolboxv2/mods/WhatsAppTb/utils.py
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
def send_initial_message(self, mode: str = "progress"):
    """
    Sends the initial message. Modes can be 'progress' or 'loading'.
    """
    if mode == "progress":
        emoji_legend = "\n".join(
            f"{emoji} - Step {i + 1}" for i, emoji in enumerate(self.emoji_set)
        )
        content = (
            "Progress is being updated in real-time!\n\n"
            "Legend:\n"
            f"{emoji_legend}\n\n"
            "Stay tuned for updates!"
        )
    elif mode == "loading":
        content = (
            "Loading in progress! 🌀\n"
            "The indicator will loop until work is done."
        )
    else:
        raise ValueError("Invalid mode. Use 'progress' or 'loading'.")

    if self.content is not None:
        content += '\n'+self.content
    message = self.messenger.create_message(content=content, to=self.recipient_phone)
    response = message.send(sender=0)
    self.message_id = response.get("messages", [{}])[0].get("id")
    logging.info(f"Initial message sent: {content}")
    return self.message_id
start_loading_in_background(stop_flag)

Starts the loading update in a separate thread.

Source code in toolboxv2/mods/WhatsAppTb/utils.py
 97
 98
 99
100
101
def start_loading_in_background(self, stop_flag: threading.Event):
    """
    Starts the loading update in a separate thread.
    """
    threading.Thread(target=self.update_loading, args=(stop_flag,), daemon=True).start()
start_progress_in_background(step_flag)

Starts the progress update in a separate thread.

Source code in toolboxv2/mods/WhatsAppTb/utils.py
91
92
93
94
95
def start_progress_in_background(self, step_flag):
    """
    Starts the progress update in a separate thread.
    """
    threading.Thread(target=self.update_progress, args=(step_flag, ), daemon=True).start()
update_loading(stop_flag)

Continuously updates the reaction to represent a looping 'loading' indicator.

Source code in toolboxv2/mods/WhatsAppTb/utils.py
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
def update_loading(self, stop_flag: threading.Event):
    """
    Continuously updates the reaction to represent a looping 'loading' indicator.
    """
    if not self.message_id:
        raise ValueError("Message ID not found. Ensure the initial message is sent first.")
    message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
    step = 0
    while not stop_flag.is_set():
        emoji = self.emoji_set[step % len(self.emoji_set)]
        message.react(emoji)
        logging.info(f"Loading update: {emoji}")
        time.sleep(1)  # Faster updates for loading
        step += 1
    # Final acknowledgment
    message.react("✅")
    logging.info("Loading completed with final acknowledgment.")
    message.reply("✅Done✅")
update_progress(step_flag)

Updates the reaction on the message to represent progress.

Source code in toolboxv2/mods/WhatsAppTb/utils.py
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def update_progress(self, step_flag: threading.Event):
    """
    Updates the reaction on the message to represent progress.
    """
    if not self.message_id:
        raise ValueError("Message ID not found. Ensure the initial message is sent first.")
    message = self.messenger.create_message(id=self.message_id, to=self.recipient_phone)
    for step in range(self.max_steps):
        emoji = self.emoji_set[step % len(self.emoji_set)]
        message.react(emoji)
        logging.info(f"Progress updated: Step {step + 1}/{self.max_steps} with emoji {emoji}")
        while not step_flag.is_set():
            time.sleep(0.5)
        step_flag.clear()
    # Final acknowledgment
    message.react("👍")
    logging.info("Progress completed with final acknowledgment.")

cli_functions

replace_bracketed_content(text, replacements, inlist=False)

Ersetzt Inhalte in eckigen Klammern mit entsprechenden Werten aus einem Wörterbuch.

:param text: Der zu verarbeitende Text als String. :param replacements: Ein Wörterbuch mit Schlüssel-Wert-Paaren für die Ersetzung. :return: Den modifizierten Text.

Source code in toolboxv2/mods/cli_functions.py
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
def replace_bracketed_content(text, replacements, inlist=False):
    """
    Ersetzt Inhalte in eckigen Klammern mit entsprechenden Werten aus einem Wörterbuch.

    :param text: Der zu verarbeitende Text als String.
    :param replacements: Ein Wörterbuch mit Schlüssel-Wert-Paaren für die Ersetzung.
    :return: Den modifizierten Text.
    """
    # Finde alle Vorkommen von Texten in eckigen Klammern
    matches = re.findall(r'\[([^\]]+)\]', text)

    # Ersetze jeden gefundenen Text durch den entsprechenden Wert aus dem Wörterbuch
    as_list = text.split(' ')
    i = 0
    for key in matches:
        if key in replacements:
            if not inlist:
                text = text.replace(f'[{key}]', str(replacements[key]))
            else:
                as_list[i] = replacements[key]
        i += 1
    if not inlist:
        return text
    return as_list

helper

create_invitation(app, username)

Creates a one-time invitation code for a user to link a new device.

Source code in toolboxv2/mods/helper.py
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
@export(mod_name=Name, name="create-invitation", test=False)
def create_invitation(app: App, username: str):
    """Creates a one-time invitation code for a user to link a new device."""
    print(f"Creating invitation for user '{username}'...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.GET_INVITATION,
                         get_results=True,
                         username=username)

    if result.is_ok():
        print(f"✅ Invitation code for '{username}': {result.get()}")
    else:
        print("❌ Error creating invitation:")
        result.print()
    return result

create_user(app, username, email)

Creates a new user with a generated key pair.

Source code in toolboxv2/mods/helper.py
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
@export(mod_name=Name, name="create-user", test=False)
def create_user(app: App, username: str, email: str):
    """Creates a new user with a generated key pair."""
    print(f"Creating user '{username}' with email '{email}'...")
    app.load_mod("CloudM")
    # Generate an invitation on the fly
    invitation_res = app.run_any(TBEF.CLOUDM_AUTHMANAGER.GET_INVITATION,
                                 get_results=True,
                                 username=username)
    if invitation_res.is_error():
        print("❌ Error creating invitation:")
        invitation_res.print()
        return invitation_res

    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.CRATE_LOCAL_ACCOUNT,
                         get_results=True,
                         username=username,
                         email=email,
                         invitation=invitation_res.get(),
                         create=True)

    if result.is_ok():
        print(f"✅ User '{username}' created successfully.")
    else:
        print("❌ Error creating user:")
        result.print()
    return result

delete_user_cli(app, username)

Deletes a user and all their associated data.

Source code in toolboxv2/mods/helper.py
85
86
87
88
89
90
91
92
93
94
95
96
@export(mod_name=Name, name="delete-user", test=False)
def delete_user_cli(app: App, username: str):
    """Deletes a user and all their associated data."""
    print(f"Attempting to delete user '{username}'...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.DELETE_USER, get_results=True, username=username)

    if result.is_ok():
        print(f"✅ User '{username}' has been deleted.")
    else:
        print(f"❌ Error deleting user: {result.info.get('help_text')}")
    return result

init_system(app) async

Initializes the ToolBoxV2 system by creating the first administrative user. This is an interactive command.

Source code in toolboxv2/mods/helper.py
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
@export(mod_name=Name, name="init_system", test=False)
async def init_system(app: App):
    """
    Initializes the ToolBoxV2 system by creating the first administrative user.
    This is an interactive command.
    """
    print("--- ToolBoxV2 System Initialization ---")
    print("This will guide you through creating the first administrator account.")
    print("This account will have the highest permission level.\n")

    try:
        username = input("Enter the administrator's username: ").strip()
        if not username:
            print("Username cannot be empty.")
            return Result.default_user_error("Username cannot be empty.")

        email = input(f"Enter the email for '{username}': ").strip()
        if not email: # A simple check, can be improved with regex
            print("Email cannot be empty.")
            return Result.default_user_error("Email cannot be empty.")

        print(f"\nCreating user '{username}' with email '{email}'...")
        # Call the internal function to create the account
        # The 'create=True' flag likely handles the initial key generation
        result = await app.a_run_any(TBEF.CLOUDM.REGISTER_INITIAL_LOOT_USER,
                                     user_name=username,
                                     email=email,
                                     get_results=True)

        if result.is_ok():
            print("\n✅ Administrator account created successfully!")
            print("   A new cryptographic key pair has been generated for this user.")
            print("   Authentication is handled automatically using these keys.")
            print("   You can now use other CLI commands or log into the web UI.")
            return Result.ok("System initialized successfully.")
        else:
            print("\n❌ Error creating administrator account:")
            result.print()
            return result

    except (KeyboardInterrupt, EOFError):
        print("\n\nInitialization cancelled by user.")
        return Result.default_user_error("Initialization cancelled.")
    except Exception as e:
        print(f"\nAn unexpected error occurred: {e}")
        return Result.default_internal_error(f"An unexpected error occurred: {e}")

list_users_cli(app)

Lists all registered users.

Source code in toolboxv2/mods/helper.py
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
@export(mod_name=Name, name="list-users", test=False)
def list_users_cli(app: App):
    """Lists all registered users."""
    print("Fetching user list...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.LIST_USERS, get_results=True)

    if result.is_ok():
        users = result.get()
        if not users:
            print("No users found.")
            return result

        print("--- Registered Users ---")
        # Simple table formatting
        print(f"{'Username':<25} {'Email':<30} {'Level'}")
        print("------------------------")
        for user in users:
            print(f"{user['username']:<25} {user['email']:<30} {user['level']}")
        print("------------------------")
    else:
        print("❌ Error listing users:")
        result.print()

    return result

Sends a magic login link to the user's registered email address.

Source code in toolboxv2/mods/helper.py
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
@export(mod_name=Name, name="send-magic-link", test=False)
def send_magic_link(app: App, username: str):
    """Sends a magic login link to the user's registered email address."""
    print(f"Sending magic link to user '{username}'...")
    app.load_mod("CloudM")
    result = app.run_any(TBEF.CLOUDM_AUTHMANAGER.GET_MAGIC_LINK_EMAIL,
                         get_results=True,
                         username=username)

    if result.is_ok():
        print(f"✅ Magic link sent successfully to the email address associated with '{username}'.")
    else:
        print("❌ Error sending magic link:")
        result.print()
    return result

isaa

CodingAgent

live
AsyncCodeDetector

Bases: NodeVisitor

Detect async code and top-level await

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
class AsyncCodeDetector(ast.NodeVisitor):
    """Detect async code and top-level await"""
    def __init__(self):
        self.has_async = False
        self.has_top_level_await = False
        self.await_nodes = []

    def visit_AsyncFunctionDef(self, node):
        self.has_async = True
        self.generic_visit(node)

    def visit_Await(self, node):
        self.has_async = True
        # Track all await nodes
        self.await_nodes.append(node)
        # Check if this await is at top level
        parent = node
        while hasattr(parent, 'parent'):
            parent = parent.parent
            if isinstance(parent, ast.AsyncFunctionDef | ast.FunctionDef):
                break
        else:
            self.has_top_level_await = True
        self.generic_visit(node)
CargoRustInterface

Usage :

Create interface

cargo_interface = CargoRustInterface()

Set up new project

await cargo_interface.setup_project("hello_rust")

Add a dependency

await cargo_interface.add_dependency("serde", "1.0")

Write and run some code

code = """ fn main() { println!("Hello, Rust!"); } """ result = await cargo_interface.run_code(code)

Modify code

new_function = """ fn main() { println!("Modified Hello, Rust!"); } """ await cargo_interface.modify_code(new_function, "main()")

Build and test

await cargo_interface.build() await cargo_interface.test()

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
class CargoRustInterface:
    '''Usage :
# Create interface
cargo_interface = CargoRustInterface()

# Set up new project
await cargo_interface.setup_project("hello_rust")

# Add a dependency
await cargo_interface.add_dependency("serde", "1.0")

# Write and run some code
code = """
fn main() {
    println!("Hello, Rust!");
}
"""
result = await cargo_interface.run_code(code)

# Modify code
new_function = """
fn main() {
    println!("Modified Hello, Rust!");
}
"""
await cargo_interface.modify_code(new_function, "main()")

# Build and test
await cargo_interface.build()
await cargo_interface.test()

    '''
    def __init__(self, session_dir=None, auto_remove=True):
        """Initialize the Rust/Cargo interface"""
        self.auto_remove = auto_remove
        self._session_dir = session_dir or Path.home() / '.cargo_sessions'
        self._session_dir.mkdir(exist_ok=True)
        self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')
        self.output_history = {}
        self._execution_count = 0
        self.current_project = None

    def reset(self):
        """Reset the interface state"""
        if self.auto_remove and self.current_project:
            shutil.rmtree(self.current_project, ignore_errors=True)
        self.output_history.clear()
        self._execution_count = 0
        self.current_project = None

    async def setup_project(self, name: str) -> str:
        """Set up a new Cargo project"""
        try:
            project_path = self.vfs.base_dir / name
            if project_path.exists():
                shutil.rmtree(project_path)

            result = subprocess.run(
                ['cargo', 'new', str(project_path)],
                capture_output=True,
                text=True, check=True
            )

            if result.returncode != 0:
                return f"Error creating project: {result.stderr}"

            self.current_project = project_path
            return f"Created new project at {project_path}"

        except Exception as e:
            return f"Failed to create project: {str(e)}"

    async def add_dependency(self, name: str, version: str | None = None) -> str:
        """Add a dependency to Cargo.toml"""
        if not self.current_project:
            return "No active project"

        try:
            cargo_toml = self.current_project / "Cargo.toml"
            if not cargo_toml.exists():
                return "Cargo.toml not found"

            cmd = ['cargo', 'add', name]
            if version:
                cmd.extend(['--vers', version])

            result = subprocess.run(
                cmd,
                cwd=self.current_project,
                capture_output=True,
                text=True,check=True
            )

            return result.stdout if result.returncode == 0 else f"Error: {result.stderr}"

        except Exception as e:
            return f"Failed to add dependency: {str(e)}"

    async def build(self, release: bool = False) -> str:
        """Build the project"""
        if not self.current_project:
            return "No active project"

        try:
            cmd = ['cargo', 'build']
            if release:
                cmd.append('--release')

            result = subprocess.run(
                cmd,
                cwd=self.current_project,
                capture_output=True,
                text=True
            )

            return result.stdout if result.returncode == 0 else f"Build error: {result.stderr}"

        except Exception as e:
            return f"Build failed: {str(e)}"

    async def test(self) -> str:
        """Run project tests"""
        if not self.current_project:
            return "No active project"

        try:
            result = subprocess.run(
                ['cargo', 'test'],
                cwd=self.current_project,
                capture_output=True,
                text=True, check=True
            )

            return result.stdout if result.returncode == 0 else f"Test error: {result.stderr}"

        except Exception as e:
            return f"Tests failed: {str(e)}"

    async def run_code(self, code: str) -> str:
        """Run Rust code"""
        if not self.current_project:
            return "No active project"

        try:
            # Write code to main.rs
            main_rs = self.current_project / "src" / "main.rs"
            with open(main_rs, 'w') as f:
                f.write(code)

            # Build and run
            build_result = subprocess.run(
                ['cargo', 'build'],
                cwd=self.current_project,
                capture_output=True,
                text=True
            )

            if build_result.returncode != 0:
                return f"Compilation error: {build_result.stderr}"

            run_result = subprocess.run(
                ['cargo', 'run'],
                cwd=self.current_project,
                capture_output=True,
                text=True
            )

            self._execution_count += 1
            output = {
                'code': code,
                'stdout': run_result.stdout,
                'stderr': run_result.stderr,
                'result': run_result.returncode == 0
            }
            self.output_history[self._execution_count] = output

            return run_result.stdout if run_result.returncode == 0 else f"Runtime error: {run_result.stderr}"

        except Exception as e:
            return f"Execution failed: {str(e)}"

    async def modify_code(self, code: str, object_name: str, file: str = "src/main.rs") -> str:
        """Modify existing Rust code"""
        if not self.current_project:
            return "No active project"

        try:
            file_path = self.current_project / file
            if not file_path.exists():
                return f"File {file} not found"

            with open(file_path) as f:
                content = f.read()

            # Handle function modification
            if object_name.endswith("()"):
                func_name = object_name[:-2]
                # Find and replace function definition
                pattern = f"fn {func_name}.*?}}(?=\n|$)"
                updated_content = re.sub(pattern, code.strip(), content, flags=re.DOTALL)
            else:
                # Handle other modifications (structs, constants, etc.)
                pattern = f"{object_name}.*?(?=\n|$)"
                updated_content = re.sub(pattern, code.strip(), content)

            with open(file_path, 'w') as f:
                f.write(updated_content)

            return f"Modified {object_name} in {file}"

        except Exception as e:
            return f"Modification failed: {str(e)}"

    def save_session(self, name: str):
        """Save current session state"""
        session_file = self._session_dir / f"{name}.json"
        state = {
            'output_history': self.output_history,
            'current_project': str(self.current_project) if self.current_project else None
        }

        with open(session_file, 'w') as f:
            json.dump(state, f)

    def load_session(self, name: str):
        """Load saved session state"""
        session_file = self._session_dir / f"{name}.json"
        if session_file.exists():
            with open(session_file) as f:
                state = json.load(f)
                self.output_history = state['output_history']
                self.current_project = Path(state['current_project']) if state['current_project'] else None
__init__(session_dir=None, auto_remove=True)

Initialize the Rust/Cargo interface

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
65
66
67
68
69
70
71
72
73
def __init__(self, session_dir=None, auto_remove=True):
    """Initialize the Rust/Cargo interface"""
    self.auto_remove = auto_remove
    self._session_dir = session_dir or Path.home() / '.cargo_sessions'
    self._session_dir.mkdir(exist_ok=True)
    self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')
    self.output_history = {}
    self._execution_count = 0
    self.current_project = None
add_dependency(name, version=None) async

Add a dependency to Cargo.toml

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
async def add_dependency(self, name: str, version: str | None = None) -> str:
    """Add a dependency to Cargo.toml"""
    if not self.current_project:
        return "No active project"

    try:
        cargo_toml = self.current_project / "Cargo.toml"
        if not cargo_toml.exists():
            return "Cargo.toml not found"

        cmd = ['cargo', 'add', name]
        if version:
            cmd.extend(['--vers', version])

        result = subprocess.run(
            cmd,
            cwd=self.current_project,
            capture_output=True,
            text=True,check=True
        )

        return result.stdout if result.returncode == 0 else f"Error: {result.stderr}"

    except Exception as e:
        return f"Failed to add dependency: {str(e)}"
build(release=False) async

Build the project

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
async def build(self, release: bool = False) -> str:
    """Build the project"""
    if not self.current_project:
        return "No active project"

    try:
        cmd = ['cargo', 'build']
        if release:
            cmd.append('--release')

        result = subprocess.run(
            cmd,
            cwd=self.current_project,
            capture_output=True,
            text=True
        )

        return result.stdout if result.returncode == 0 else f"Build error: {result.stderr}"

    except Exception as e:
        return f"Build failed: {str(e)}"
load_session(name)

Load saved session state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
257
258
259
260
261
262
263
264
def load_session(self, name: str):
    """Load saved session state"""
    session_file = self._session_dir / f"{name}.json"
    if session_file.exists():
        with open(session_file) as f:
            state = json.load(f)
            self.output_history = state['output_history']
            self.current_project = Path(state['current_project']) if state['current_project'] else None
modify_code(code, object_name, file='src/main.rs') async

Modify existing Rust code

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
async def modify_code(self, code: str, object_name: str, file: str = "src/main.rs") -> str:
    """Modify existing Rust code"""
    if not self.current_project:
        return "No active project"

    try:
        file_path = self.current_project / file
        if not file_path.exists():
            return f"File {file} not found"

        with open(file_path) as f:
            content = f.read()

        # Handle function modification
        if object_name.endswith("()"):
            func_name = object_name[:-2]
            # Find and replace function definition
            pattern = f"fn {func_name}.*?}}(?=\n|$)"
            updated_content = re.sub(pattern, code.strip(), content, flags=re.DOTALL)
        else:
            # Handle other modifications (structs, constants, etc.)
            pattern = f"{object_name}.*?(?=\n|$)"
            updated_content = re.sub(pattern, code.strip(), content)

        with open(file_path, 'w') as f:
            f.write(updated_content)

        return f"Modified {object_name} in {file}"

    except Exception as e:
        return f"Modification failed: {str(e)}"
reset()

Reset the interface state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
75
76
77
78
79
80
81
def reset(self):
    """Reset the interface state"""
    if self.auto_remove and self.current_project:
        shutil.rmtree(self.current_project, ignore_errors=True)
    self.output_history.clear()
    self._execution_count = 0
    self.current_project = None
run_code(code) async

Run Rust code

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
async def run_code(self, code: str) -> str:
    """Run Rust code"""
    if not self.current_project:
        return "No active project"

    try:
        # Write code to main.rs
        main_rs = self.current_project / "src" / "main.rs"
        with open(main_rs, 'w') as f:
            f.write(code)

        # Build and run
        build_result = subprocess.run(
            ['cargo', 'build'],
            cwd=self.current_project,
            capture_output=True,
            text=True
        )

        if build_result.returncode != 0:
            return f"Compilation error: {build_result.stderr}"

        run_result = subprocess.run(
            ['cargo', 'run'],
            cwd=self.current_project,
            capture_output=True,
            text=True
        )

        self._execution_count += 1
        output = {
            'code': code,
            'stdout': run_result.stdout,
            'stderr': run_result.stderr,
            'result': run_result.returncode == 0
        }
        self.output_history[self._execution_count] = output

        return run_result.stdout if run_result.returncode == 0 else f"Runtime error: {run_result.stderr}"

    except Exception as e:
        return f"Execution failed: {str(e)}"
save_session(name)

Save current session state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
246
247
248
249
250
251
252
253
254
255
def save_session(self, name: str):
    """Save current session state"""
    session_file = self._session_dir / f"{name}.json"
    state = {
        'output_history': self.output_history,
        'current_project': str(self.current_project) if self.current_project else None
    }

    with open(session_file, 'w') as f:
        json.dump(state, f)
setup_project(name) async

Set up a new Cargo project

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
async def setup_project(self, name: str) -> str:
    """Set up a new Cargo project"""
    try:
        project_path = self.vfs.base_dir / name
        if project_path.exists():
            shutil.rmtree(project_path)

        result = subprocess.run(
            ['cargo', 'new', str(project_path)],
            capture_output=True,
            text=True, check=True
        )

        if result.returncode != 0:
            return f"Error creating project: {result.stderr}"

        self.current_project = project_path
        return f"Created new project at {project_path}"

    except Exception as e:
        return f"Failed to create project: {str(e)}"
test() async

Run project tests

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
async def test(self) -> str:
    """Run project tests"""
    if not self.current_project:
        return "No active project"

    try:
        result = subprocess.run(
            ['cargo', 'test'],
            cwd=self.current_project,
            capture_output=True,
            text=True, check=True
        )

        return result.stdout if result.returncode == 0 else f"Test error: {result.stderr}"

    except Exception as e:
        return f"Tests failed: {str(e)}"
MockIPython
Source code in toolboxv2/mods/isaa/CodingAgent/live.py
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
class MockIPython:
    def __init__(self, _session_dir=None, auto_remove=True):
        self.auto_remove = auto_remove
        self.output_history = {}
        self._execution_count = 0
        self._session_dir = _session_dir or Path(get_app().appdata) / '.pipeline_sessions'
        self._session_dir.mkdir(exist_ok=True)
        self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')
        self._venv_path = self._session_dir / 'venv'
        self.user_ns: dict[str, Any] = {}
        # import nest_asyncio
        # nest_asyncio.apply()
        # Set up virtual environment if it doesn't exist
        with Spinner("Starting virtual environment"):
            self._setup_venv()
        self.reset()

    def _virtual_open(self, filepath, mode='r', *args, **kwargs):
        """Custom open function that uses virtual filesystem and makes files available for import"""
        try:
            abs_path = self.vfs._resolve_path(filepath)
        except ValueError:
            # If path resolution fails, try to resolve relative to current working directory
            abs_path = self.vfs.base_dir / filepath

        if 'w' in mode or 'a' in mode:
            # Ensure parent directory exists
            abs_path.parent.mkdir(parents=True, exist_ok=True)

        # Use actual filesystem but track in virtual fs
        real_file = open(abs_path, mode, *args, **kwargs)

        if 'r' in mode:
            # Track file content in virtual filesystem when reading
            rel_path = str(abs_path.relative_to(self.vfs.base_dir))
            if rel_path not in self.vfs.virtual_files:
                try:
                    content = real_file.read()
                    self.vfs.virtual_files[rel_path] = content
                    real_file.seek(0)
                except UnicodeDecodeError:
                    # Handle binary files
                    pass

        return real_file

    def _setup_venv(self):
        """Create virtual environment if it doesn't exist"""
        if not self._venv_path.exists():
            try:
                subprocess.run([sys.executable, "-m", "venv", str(self._venv_path)], check=True)
            except subprocess.CalledProcessError as e:
                raise RuntimeError(f"Failed to create virtual environment: {str(e)}")

    def _virtual_open(self, filepath, mode='r', *args, **kwargs):
        """Custom open function that uses virtual filesystem"""
        abs_path = self.vfs._resolve_path(filepath)

        if 'w' in mode or 'a' in mode:
            # Ensure parent directory exists
            abs_path.parent.mkdir(parents=True, exist_ok=True)

        # Use actual filesystem but track in virtual fs
        real_file = open(abs_path, mode, *args, **kwargs)

        if 'r' in mode:
            # Track file content in virtual filesystem when reading
            rel_path = str(abs_path.relative_to(self.vfs.base_dir))
            if rel_path not in self.vfs.virtual_files:
                try:
                    self.vfs.virtual_files[rel_path] = real_file.read()
                    real_file.seek(0)
                except UnicodeDecodeError:
                    # Handle binary files
                    pass

        return real_file

    def reset(self):
        """Reset the interpreter state"""
        self.user_ns = {
            '__name__': '__main__',
            '__builtins__': __builtins__,
            'toolboxv2': toolboxv2,
            '__file__': None,
            '__path__': [str(self.vfs.current_dir)],
            'auto_install': auto_install,
            'app': get_app(),
            'modify_code': self.modify_code,
            'open': self._virtual_open,
        }
        self.output_history.clear()
        self._execution_count = 0
        if self.auto_remove:
            shutil.rmtree(self.vfs.base_dir, ignore_errors=True)

    def get_namespace(self) -> dict[str, Any]:
        """Get current namespace"""
        return self.user_ns.copy()

    def update_namespace(self, variables: dict[str, Any]):
        """Update namespace with new variables"""
        self.user_ns.update(variables)

    @staticmethod
    def _parse_code(code: str) -> tuple[Any, Any | None, bool, bool]:
        """Parse code and handle top-level await"""
        code_ = ""
        for line in code.split('\n'):
            if line.strip().startswith('#'):
                continue
            if line.strip().startswith('asyncio.run('):
                line = (' ' *(len(line) - len(line.strip()))) + 'await ' + line.strip()[len('asyncio.run('):-1]
            code_ += line + '\n'
        try:
            tree = ast.parse(code)
            # Add parent references
            ParentNodeTransformer().visit(tree)

            # Detect async features
            detector = AsyncCodeDetector()
            detector.visit(tree)

            if detector.has_top_level_await:
                # Wrap code in async function
                wrapped_code = "async def __wrapper():\n"
                wrapped_code += "    global result\n"  # Allow writing to global scope
                wrapped_code += "    result = None\n"
                # add try:
                wrapped_code +="    try:\n"
                # Indent the original code
                wrapped_code += "\n".join(f"        {line}" for line in code.splitlines())
                # Add return statement for last expression
                wrapped_code +="\n    except Exception as e:\n"
                wrapped_code +="        import traceback\n"
                wrapped_code +="        print(traceback.format_exc())\n"
                wrapped_code +="        raise e\n"
                if isinstance(tree.body[-1], ast.Expr):
                    wrapped_code += "\n    return result"

                # Parse and compile wrapped code
                wrapped_tree = ast.parse(wrapped_code)
                return (
                    compile(wrapped_tree, '<exec>', 'exec'),
                    None,
                    True,
                    True
                )

            # Handle regular code
            if isinstance(tree.body[-1], ast.Expr):
                exec_code = ast.Module(
                    body=tree.body[:-1],
                    type_ignores=[]
                )
                eval_code = ast.Expression(
                    body=tree.body[-1].value
                )
                return (
                    compile(exec_code, '<exec>', 'exec'),
                    compile(eval_code, '<eval>', 'eval'),
                    detector.has_async,
                    False
                )

            return (
                compile(tree, '<exec>', 'exec'),
                None,
                detector.has_async,
                False
            )

        except SyntaxError as e:
            lines = code.splitlines()
            if e.lineno and e.lineno <= len(lines):
                line = lines[e.lineno - 1]
                arrow = ' ' * (e.offset - 1) + '^' if e.offset else ''
                error_msg = (
                    f"Syntax error at line {e.lineno}:\n"
                    f"{line}\n"
                    f"{arrow}\n"
                    f"{e.msg}"
                )
            else:
                error_msg = str(e)

            error_msg += traceback.format_exc()

            raise SyntaxError(error_msg) from e

    async def run_cell(self, code: str, live_output: bool = True) -> Any:
        """Async version of run_cell that handles both sync and async code"""
        result = None
        error = None
        tb = None
        original_dir = os.getcwd()

        if live_output:
            stdout_buffer = io.StringIO()
            stderr_buffer = io.StringIO()
            stdout = TeeStream(sys.__stdout__, stdout_buffer)
            stderr = TeeStream(sys.__stderr__, stderr_buffer)
        else:
            stdout = io.StringIO()
            stderr = io.StringIO()

        try:
            # Check if a file is already specified
            original_file = self.user_ns.get('__file__')
            if original_file is None:
                # Create temp file if no file specified
                temp_file = self.vfs.write_file(
                    f'src/temp/_temp_{self._execution_count}.py',
                    code
                )
                # work_ns = self.user_ns.copy()
                self.user_ns['__file__'] = str(temp_file)
            else:
                # Use existing file
                temp_file = Path(original_file)
                # Write code to the existing file
                self.vfs.write_file(temp_file, code)
                #work_ns = self.user_ns.copy()

            self.user_ns['__builtins__'] = __builtins__
            with VirtualEnvContext(self._venv_path) as python_exec:
                try:
                    exec_code, eval_code, is_async, has_top_level_await = self._parse_code(
                        code.encode('utf-8', errors='replace').decode('utf-8')
                    )
                    if exec_code is None:
                        return "No executable code"
                    os.makedirs(str(temp_file.parent.absolute()), exist_ok=True)
                    os.chdir(str(temp_file.parent.absolute()))
                    self.user_ns['PYTHON_EXEC'] = python_exec

                    with redirect_stdout(stdout), redirect_stderr(stderr):
                        if has_top_level_await:
                            try:
                                # Execute wrapped code and await it
                                exec(exec_code, self.user_ns)
                                result = self.user_ns['__wrapper']()
                                if asyncio.iscoroutine(result):
                                    result = await result
                            finally:
                                self.user_ns.pop('__wrapper', None)
                        elif is_async:
                            # Execute async code
                            exec(exec_code, self.user_ns)
                            if eval_code:
                                result = eval(eval_code, self.user_ns)
                                if asyncio.iscoroutine(result):
                                    result = await result
                        else:
                            # Execute sync code
                            exec(exec_code, self.user_ns)
                            if eval_code:
                                result = eval(eval_code, self.user_ns)

                        if result is not None:
                            self.user_ns['_'] = result
                except KeyboardInterrupt:
                    print("Stop execution manuel!")

                except Exception as e:
                    error = str(e)
                    tb = traceback.format_exc()
                    if live_output:
                        sys.__stderr__.write(f"{error}\n{tb}")
                    stderr.write(f"{error}\n{tb}")

                finally:
                    os.chdir(original_dir)
                    self._execution_count += 1
                    # self.user_ns = work_ns.copy()
                    if live_output:
                        stdout_value = stdout_buffer.getvalue()
                        stderr_value = stderr_buffer.getvalue()
                    else:
                        stdout_value = stdout.getvalue()
                        stderr_value = stderr.getvalue()

                    output = {
                        'code': code,
                        'stdout': stdout_value,
                        'stderr': stderr_value,
                        'result': result if result else "stdout"
                    }
                    self.output_history[self._execution_count] = output

        except Exception as e:
            error_msg = f"Error executing code: {str(e)}\n{traceback.format_exc()}"
            if live_output:
                sys.__stderr__.write(error_msg)
            return error_msg

        if not result:
            result = ""
        if output['stdout']:
            result = f"{result}\nstdout:{output['stdout']}"
        if output['stderr']:
            result = f"{result}\nstderr:{output['stderr']}"

        if self.auto_remove and original_file is None:
            # Only remove temp files, not user-specified files
            self.vfs.delete_file(temp_file)

        return result

    async def modify_code(self, code: str = None, object_name: str = None, file: str = None) -> str:
        '''
        Modify existing code in memory (user namespace) and optionally in the corresponding file.

        This method updates variables, functions, or methods in the current Python session and can
        also update the corresponding source file if specified.

        Args:
            code: New value or implementation for the object
            object_name: Name of the object to modify (variable, function, or method)
            file: Path to the file to update (if None, only updates in memory)

        Returns:
            String describing the modification result

        Examples:

        # 1. Update a variable in memory
        await ipython.modify_code(code="5", object_name="x")

    # 2. Change a method implementation
    await ipython.modify_code(
        code='"""def sound(self):\n        return "Woof""""',
        object_name="Dog.sound"
    )

    # 3. Modify a function
    await ipython.modify_code(
        code='"""def calculate_age():\n    return 25"""',
        object_name="calculate_age"
    )

    # 4. Update variable in memory and file
    await ipython.modify_code(
        code="100",
        object_name="MAX_SIZE",
        file="config.py"
    )

    # 5. Modifying an attribute in __init__
    await ipython.modify_code(
        code='"""def __init__(self):\n        self.name = "Buddy""""',
        object_name="Dog.__init__"
    )
        '''
        try:
            if not object_name:
                raise ValueError("Object name must be specified")
            if code is None:
                raise ValueError("New code or value must be provided")

            # Process object name (handle methods with parentheses)
            clean_object_name = object_name.replace("()", "")

            # Step 1: Update in memory (user namespace)
            result_message = []

            # Handle different types of objects
            if "." in clean_object_name:
                # For methods or class attributes
                parts = clean_object_name.split(".")
                base_obj_name = parts[0]
                attr_name = parts[1]

                if base_obj_name not in self.user_ns:
                    raise ValueError(f"Object '{base_obj_name}' not found in namespace")

                base_obj = self.user_ns[base_obj_name]

                # Handle method definitions which are passed as docstrings
                if code.split('\n'):
                    method_code = code

                    # Parse the method code to extract its body
                    method_ast = ast.parse(method_code).body[0]
                    method_name = method_ast.name

                    # Create a new function object from the code
                    method_locals = {}
                    exec(
                        f"def _temp_func{signature(getattr(base_obj.__class__, attr_name, None))}: {method_ast.body[0].value.s}",
                        globals(), method_locals)
                    new_method = method_locals['_temp_func']

                    # Set the method on the class
                    setattr(base_obj.__class__, attr_name, new_method)
                    result_message.append(f"Updated method '{clean_object_name}' in memory")
                else:
                    # For simple attributes
                    setattr(base_obj, attr_name, eval(code, self.user_ns))
                    result_message.append(f"Updated attribute '{clean_object_name}' in memory")
            else:
                # For variables and functions
                if code.startswith('"""') and code.endswith('"""'):
                    # Handle function definitions
                    func_code = code.strip('"""')
                    func_ast = ast.parse(func_code).body[0]
                    func_name = func_ast.name

                    # Create a new function object from the code
                    func_locals = {}
                    exec(f"{func_code}", globals(), func_locals)
                    self.user_ns[clean_object_name] = func_locals[func_name]
                    result_message.append(f"Updated function '{clean_object_name}' in memory")
                else:
                    # Simple variable assignment
                    self.user_ns[clean_object_name] = eval(code, self.user_ns)
                    result_message.append(f"Updated variable '{clean_object_name}' in memory")

            # Step 2: Update in file if specified
            if file is not None:
                file_path = self.vfs._resolve_path(file)

                if not file_path.exists():
                    self.user_ns['__file__'] = str(file_path)
                    return await self.run_cell(code)

                # Read original content
                original_content = self.vfs.read_file(file_path)
                updated_content = original_content

                # Handle different object types for file updates
                if "." in clean_object_name:
                    # For methods
                    parts = clean_object_name.split(".")
                    class_name = parts[0]
                    method_name = parts[1]

                    if code.startswith('"""') and code.endswith('"""'):
                        method_code = code.strip('"""')

                        # Use ast to parse the file and find the method to replace
                        file_ast = ast.parse(original_content)
                        for node in ast.walk(file_ast):
                            if isinstance(node, ast.ClassDef) and node.name == class_name:
                                for method in node.body:
                                    if isinstance(method, ast.FunctionDef) and method.name == method_name:
                                        # Find the method in the source code
                                        method_pattern = fr"def {method_name}.*?:(.*?)(?=\n    \w|\n\w|\Z)"
                                        method_match = re.search(method_pattern, original_content, re.DOTALL)

                                        if method_match:
                                            indentation = re.match(r"^(\s*)", method_match.group(0)).group(1)
                                            method_indented = textwrap.indent(method_code, indentation)
                                            updated_content = original_content.replace(
                                                method_match.group(0),
                                                method_indented
                                            )
                                            self.vfs.write_file(file_path, updated_content)
                                            result_message.append(
                                                f"Updated method '{clean_object_name}' in file '{file}'")
                else:
                    # For variables and functions
                    if code.startswith('"""') and code.endswith('"""'):
                        # Handle function updates
                        func_code = code.strip('"""')
                        func_pattern = fr"def {clean_object_name}.*?:(.*?)(?=\n\w|\Z)"
                        func_match = re.search(func_pattern, original_content, re.DOTALL)

                        if func_match:
                            indentation = re.match(r"^(\s*)", func_match.group(0)).group(1)
                            func_indented = textwrap.indent(func_code, indentation)
                            updated_content = original_content.replace(
                                func_match.group(0),
                                func_indented
                            )
                            self.vfs.write_file(file_path, updated_content)
                            result_message.append(f"Updated function '{clean_object_name}' in file '{file}'")
                    else:
                        # Handle variable updates
                        var_pattern = fr"{clean_object_name}\s*=.*"
                        var_replacement = f"{clean_object_name} = {code}"
                        updated_content = re.sub(var_pattern, var_replacement, original_content)

                        if updated_content != original_content:
                            self.vfs.write_file(file_path, updated_content)
                            result_message.append(f"Updated variable '{clean_object_name}' in file '{file}'")
                        else:
                            result_message.append(f"Could not find variable '{clean_object_name}' in file '{file}'")

            return "\n".join(result_message)

        except Exception as e:
            return f"Error during code modification: {str(e)}\n{traceback.format_exc()}"


    def save_session(self, name: str):
        """Save session with UTF-8 encoding"""
        session_file = self._session_dir / f"{name}.pkl"
        user_ns = self.user_ns.copy()
        output_history = self.output_history.copy()

        # Ensure all strings are properly encoded
        for key, value in user_ns.items():
            try:
                if isinstance(value, str):
                    value = value.encode('utf-8').decode('utf-8')
                pickle.dumps(value)
            except Exception:
                user_ns[key] = f"not serializable: {str(value)}"

        for key, value in output_history.items():
            try:
                if isinstance(value, dict):
                    for k, v in value.items():
                        if isinstance(v, str):
                            value[k] = v.encode('utf-8').decode('utf-8')
                pickle.dumps(value)
            except Exception:
                output_history[key] = f"not serializable: {str(value)}"


        session_data = {
            'user_ns': user_ns,
            'output_history': output_history,

        }

        with open(session_file, 'wb') as f:
            pickle.dump(session_data, f)

        # Save VFS state with UTF-8 encoding
        vfs_state_file = self._session_dir / f"{name}_vfs.json"
        with open(vfs_state_file, 'w', encoding='utf-8') as f:
            json.dump(self.vfs.virtual_files, f, ensure_ascii=False)

    def load_session(self, name: str):
        """Load session with UTF-8 encoding"""
        session_file = self._session_dir / f"{name}.pkl"
        if session_file.exists():
            with open(session_file, 'rb') as f:
                session_data = pickle.load(f)
                # self.user_ns.update(session_data['user_ns'])
                self.output_history.update(session_data['output_history'])

        # Load VFS state with UTF-8 encoding
        vfs_state_file = self._session_dir / f"{name}_vfs.json"
        if vfs_state_file.exists():
            with open(vfs_state_file, encoding='utf-8') as f:
                self.vfs.virtual_files = json.load(f)

    def __str__(self):
        """String representation of current session"""
        output = []
        for count, data in self.output_history.items():
            output.append(f"In [{count}]: {data['code']}")
            if data['stdout']:
                output.append(data['stdout'])
            if data['stderr']:
                output.append(f"Error: {data['stderr']}")
            if data['result'] is not None:
                output.append(f"Out[{count}]: {data['result']}")
        return "\n".join(output)

    def set_base_directory(self, path: str) -> str:
        """
        Set the base directory for the virtual file system and add it to sys.path for imports.

        Args:
            path: New base directory path

        Returns:
            Success message
        """
        try:
            new_path = Path(path) if isinstance(path, str) else path
            new_path.mkdir(parents=True, exist_ok=True)

            # Remove old base directory from sys.path if it exists
            old_base_str = str(self.vfs.base_dir)
            if old_base_str in sys.path:
                sys.path.remove(old_base_str)

            # Update VFS base directory
            self.vfs.base_dir = new_path
            self.vfs.current_dir = new_path

            # Add new base directory to sys.path for imports
            new_base_str = str(new_path)
            if new_base_str not in sys.path:
                sys.path.insert(0, new_base_str)

            # Update user namespace paths
            self.user_ns['__path__'] = [new_base_str]

            return f"Base directory set to: {new_path} (added to sys.path)"

        except Exception as e:
            return f"Set base directory error: {str(e)}"
__str__()

String representation of current session

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
def __str__(self):
    """String representation of current session"""
    output = []
    for count, data in self.output_history.items():
        output.append(f"In [{count}]: {data['code']}")
        if data['stdout']:
            output.append(data['stdout'])
        if data['stderr']:
            output.append(f"Error: {data['stderr']}")
        if data['result'] is not None:
            output.append(f"Out[{count}]: {data['result']}")
    return "\n".join(output)
get_namespace()

Get current namespace

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
672
673
674
def get_namespace(self) -> dict[str, Any]:
    """Get current namespace"""
    return self.user_ns.copy()
load_session(name)

Load session with UTF-8 encoding

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
def load_session(self, name: str):
    """Load session with UTF-8 encoding"""
    session_file = self._session_dir / f"{name}.pkl"
    if session_file.exists():
        with open(session_file, 'rb') as f:
            session_data = pickle.load(f)
            # self.user_ns.update(session_data['user_ns'])
            self.output_history.update(session_data['output_history'])

    # Load VFS state with UTF-8 encoding
    vfs_state_file = self._session_dir / f"{name}_vfs.json"
    if vfs_state_file.exists():
        with open(vfs_state_file, encoding='utf-8') as f:
            self.vfs.virtual_files = json.load(f)
modify_code(code=None, object_name=None, file=None) async
Modify existing code in memory (user namespace) and optionally in the corresponding file.

This method updates variables, functions, or methods in the current Python session and can
also update the corresponding source file if specified.

Args:
    code: New value or implementation for the object
    object_name: Name of the object to modify (variable, function, or method)
    file: Path to the file to update (if None, only updates in memory)

Returns:
    String describing the modification result

Examples:

# 1. Update a variable in memory
await ipython.modify_code(code="5", object_name="x")
2. Change a method implementation

await ipython.modify_code( code='"""def sound(self): return "Woof""""', object_name="Dog.sound" )

3. Modify a function

await ipython.modify_code( code='"""def calculate_age(): return 25"""', object_name="calculate_age" )

4. Update variable in memory and file

await ipython.modify_code( code="100", object_name="MAX_SIZE", file="config.py" )

5. Modifying an attribute in init

await ipython.modify_code( code='"""def init(self): self.name = "Buddy""""', object_name="Dog.init" )

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
async def modify_code(self, code: str = None, object_name: str = None, file: str = None) -> str:
    '''
    Modify existing code in memory (user namespace) and optionally in the corresponding file.

    This method updates variables, functions, or methods in the current Python session and can
    also update the corresponding source file if specified.

    Args:
        code: New value or implementation for the object
        object_name: Name of the object to modify (variable, function, or method)
        file: Path to the file to update (if None, only updates in memory)

    Returns:
        String describing the modification result

    Examples:

    # 1. Update a variable in memory
    await ipython.modify_code(code="5", object_name="x")

# 2. Change a method implementation
await ipython.modify_code(
    code='"""def sound(self):\n        return "Woof""""',
    object_name="Dog.sound"
)

# 3. Modify a function
await ipython.modify_code(
    code='"""def calculate_age():\n    return 25"""',
    object_name="calculate_age"
)

# 4. Update variable in memory and file
await ipython.modify_code(
    code="100",
    object_name="MAX_SIZE",
    file="config.py"
)

# 5. Modifying an attribute in __init__
await ipython.modify_code(
    code='"""def __init__(self):\n        self.name = "Buddy""""',
    object_name="Dog.__init__"
)
    '''
    try:
        if not object_name:
            raise ValueError("Object name must be specified")
        if code is None:
            raise ValueError("New code or value must be provided")

        # Process object name (handle methods with parentheses)
        clean_object_name = object_name.replace("()", "")

        # Step 1: Update in memory (user namespace)
        result_message = []

        # Handle different types of objects
        if "." in clean_object_name:
            # For methods or class attributes
            parts = clean_object_name.split(".")
            base_obj_name = parts[0]
            attr_name = parts[1]

            if base_obj_name not in self.user_ns:
                raise ValueError(f"Object '{base_obj_name}' not found in namespace")

            base_obj = self.user_ns[base_obj_name]

            # Handle method definitions which are passed as docstrings
            if code.split('\n'):
                method_code = code

                # Parse the method code to extract its body
                method_ast = ast.parse(method_code).body[0]
                method_name = method_ast.name

                # Create a new function object from the code
                method_locals = {}
                exec(
                    f"def _temp_func{signature(getattr(base_obj.__class__, attr_name, None))}: {method_ast.body[0].value.s}",
                    globals(), method_locals)
                new_method = method_locals['_temp_func']

                # Set the method on the class
                setattr(base_obj.__class__, attr_name, new_method)
                result_message.append(f"Updated method '{clean_object_name}' in memory")
            else:
                # For simple attributes
                setattr(base_obj, attr_name, eval(code, self.user_ns))
                result_message.append(f"Updated attribute '{clean_object_name}' in memory")
        else:
            # For variables and functions
            if code.startswith('"""') and code.endswith('"""'):
                # Handle function definitions
                func_code = code.strip('"""')
                func_ast = ast.parse(func_code).body[0]
                func_name = func_ast.name

                # Create a new function object from the code
                func_locals = {}
                exec(f"{func_code}", globals(), func_locals)
                self.user_ns[clean_object_name] = func_locals[func_name]
                result_message.append(f"Updated function '{clean_object_name}' in memory")
            else:
                # Simple variable assignment
                self.user_ns[clean_object_name] = eval(code, self.user_ns)
                result_message.append(f"Updated variable '{clean_object_name}' in memory")

        # Step 2: Update in file if specified
        if file is not None:
            file_path = self.vfs._resolve_path(file)

            if not file_path.exists():
                self.user_ns['__file__'] = str(file_path)
                return await self.run_cell(code)

            # Read original content
            original_content = self.vfs.read_file(file_path)
            updated_content = original_content

            # Handle different object types for file updates
            if "." in clean_object_name:
                # For methods
                parts = clean_object_name.split(".")
                class_name = parts[0]
                method_name = parts[1]

                if code.startswith('"""') and code.endswith('"""'):
                    method_code = code.strip('"""')

                    # Use ast to parse the file and find the method to replace
                    file_ast = ast.parse(original_content)
                    for node in ast.walk(file_ast):
                        if isinstance(node, ast.ClassDef) and node.name == class_name:
                            for method in node.body:
                                if isinstance(method, ast.FunctionDef) and method.name == method_name:
                                    # Find the method in the source code
                                    method_pattern = fr"def {method_name}.*?:(.*?)(?=\n    \w|\n\w|\Z)"
                                    method_match = re.search(method_pattern, original_content, re.DOTALL)

                                    if method_match:
                                        indentation = re.match(r"^(\s*)", method_match.group(0)).group(1)
                                        method_indented = textwrap.indent(method_code, indentation)
                                        updated_content = original_content.replace(
                                            method_match.group(0),
                                            method_indented
                                        )
                                        self.vfs.write_file(file_path, updated_content)
                                        result_message.append(
                                            f"Updated method '{clean_object_name}' in file '{file}'")
            else:
                # For variables and functions
                if code.startswith('"""') and code.endswith('"""'):
                    # Handle function updates
                    func_code = code.strip('"""')
                    func_pattern = fr"def {clean_object_name}.*?:(.*?)(?=\n\w|\Z)"
                    func_match = re.search(func_pattern, original_content, re.DOTALL)

                    if func_match:
                        indentation = re.match(r"^(\s*)", func_match.group(0)).group(1)
                        func_indented = textwrap.indent(func_code, indentation)
                        updated_content = original_content.replace(
                            func_match.group(0),
                            func_indented
                        )
                        self.vfs.write_file(file_path, updated_content)
                        result_message.append(f"Updated function '{clean_object_name}' in file '{file}'")
                else:
                    # Handle variable updates
                    var_pattern = fr"{clean_object_name}\s*=.*"
                    var_replacement = f"{clean_object_name} = {code}"
                    updated_content = re.sub(var_pattern, var_replacement, original_content)

                    if updated_content != original_content:
                        self.vfs.write_file(file_path, updated_content)
                        result_message.append(f"Updated variable '{clean_object_name}' in file '{file}'")
                    else:
                        result_message.append(f"Could not find variable '{clean_object_name}' in file '{file}'")

        return "\n".join(result_message)

    except Exception as e:
        return f"Error during code modification: {str(e)}\n{traceback.format_exc()}"
reset()

Reset the interpreter state

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
def reset(self):
    """Reset the interpreter state"""
    self.user_ns = {
        '__name__': '__main__',
        '__builtins__': __builtins__,
        'toolboxv2': toolboxv2,
        '__file__': None,
        '__path__': [str(self.vfs.current_dir)],
        'auto_install': auto_install,
        'app': get_app(),
        'modify_code': self.modify_code,
        'open': self._virtual_open,
    }
    self.output_history.clear()
    self._execution_count = 0
    if self.auto_remove:
        shutil.rmtree(self.vfs.base_dir, ignore_errors=True)
run_cell(code, live_output=True) async

Async version of run_cell that handles both sync and async code

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
async def run_cell(self, code: str, live_output: bool = True) -> Any:
    """Async version of run_cell that handles both sync and async code"""
    result = None
    error = None
    tb = None
    original_dir = os.getcwd()

    if live_output:
        stdout_buffer = io.StringIO()
        stderr_buffer = io.StringIO()
        stdout = TeeStream(sys.__stdout__, stdout_buffer)
        stderr = TeeStream(sys.__stderr__, stderr_buffer)
    else:
        stdout = io.StringIO()
        stderr = io.StringIO()

    try:
        # Check if a file is already specified
        original_file = self.user_ns.get('__file__')
        if original_file is None:
            # Create temp file if no file specified
            temp_file = self.vfs.write_file(
                f'src/temp/_temp_{self._execution_count}.py',
                code
            )
            # work_ns = self.user_ns.copy()
            self.user_ns['__file__'] = str(temp_file)
        else:
            # Use existing file
            temp_file = Path(original_file)
            # Write code to the existing file
            self.vfs.write_file(temp_file, code)
            #work_ns = self.user_ns.copy()

        self.user_ns['__builtins__'] = __builtins__
        with VirtualEnvContext(self._venv_path) as python_exec:
            try:
                exec_code, eval_code, is_async, has_top_level_await = self._parse_code(
                    code.encode('utf-8', errors='replace').decode('utf-8')
                )
                if exec_code is None:
                    return "No executable code"
                os.makedirs(str(temp_file.parent.absolute()), exist_ok=True)
                os.chdir(str(temp_file.parent.absolute()))
                self.user_ns['PYTHON_EXEC'] = python_exec

                with redirect_stdout(stdout), redirect_stderr(stderr):
                    if has_top_level_await:
                        try:
                            # Execute wrapped code and await it
                            exec(exec_code, self.user_ns)
                            result = self.user_ns['__wrapper']()
                            if asyncio.iscoroutine(result):
                                result = await result
                        finally:
                            self.user_ns.pop('__wrapper', None)
                    elif is_async:
                        # Execute async code
                        exec(exec_code, self.user_ns)
                        if eval_code:
                            result = eval(eval_code, self.user_ns)
                            if asyncio.iscoroutine(result):
                                result = await result
                    else:
                        # Execute sync code
                        exec(exec_code, self.user_ns)
                        if eval_code:
                            result = eval(eval_code, self.user_ns)

                    if result is not None:
                        self.user_ns['_'] = result
            except KeyboardInterrupt:
                print("Stop execution manuel!")

            except Exception as e:
                error = str(e)
                tb = traceback.format_exc()
                if live_output:
                    sys.__stderr__.write(f"{error}\n{tb}")
                stderr.write(f"{error}\n{tb}")

            finally:
                os.chdir(original_dir)
                self._execution_count += 1
                # self.user_ns = work_ns.copy()
                if live_output:
                    stdout_value = stdout_buffer.getvalue()
                    stderr_value = stderr_buffer.getvalue()
                else:
                    stdout_value = stdout.getvalue()
                    stderr_value = stderr.getvalue()

                output = {
                    'code': code,
                    'stdout': stdout_value,
                    'stderr': stderr_value,
                    'result': result if result else "stdout"
                }
                self.output_history[self._execution_count] = output

    except Exception as e:
        error_msg = f"Error executing code: {str(e)}\n{traceback.format_exc()}"
        if live_output:
            sys.__stderr__.write(error_msg)
        return error_msg

    if not result:
        result = ""
    if output['stdout']:
        result = f"{result}\nstdout:{output['stdout']}"
    if output['stderr']:
        result = f"{result}\nstderr:{output['stderr']}"

    if self.auto_remove and original_file is None:
        # Only remove temp files, not user-specified files
        self.vfs.delete_file(temp_file)

    return result
save_session(name)

Save session with UTF-8 encoding

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
def save_session(self, name: str):
    """Save session with UTF-8 encoding"""
    session_file = self._session_dir / f"{name}.pkl"
    user_ns = self.user_ns.copy()
    output_history = self.output_history.copy()

    # Ensure all strings are properly encoded
    for key, value in user_ns.items():
        try:
            if isinstance(value, str):
                value = value.encode('utf-8').decode('utf-8')
            pickle.dumps(value)
        except Exception:
            user_ns[key] = f"not serializable: {str(value)}"

    for key, value in output_history.items():
        try:
            if isinstance(value, dict):
                for k, v in value.items():
                    if isinstance(v, str):
                        value[k] = v.encode('utf-8').decode('utf-8')
            pickle.dumps(value)
        except Exception:
            output_history[key] = f"not serializable: {str(value)}"


    session_data = {
        'user_ns': user_ns,
        'output_history': output_history,

    }

    with open(session_file, 'wb') as f:
        pickle.dump(session_data, f)

    # Save VFS state with UTF-8 encoding
    vfs_state_file = self._session_dir / f"{name}_vfs.json"
    with open(vfs_state_file, 'w', encoding='utf-8') as f:
        json.dump(self.vfs.virtual_files, f, ensure_ascii=False)
set_base_directory(path)

Set the base directory for the virtual file system and add it to sys.path for imports.

Parameters:

Name Type Description Default
path str

New base directory path

required

Returns:

Type Description
str

Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
def set_base_directory(self, path: str) -> str:
    """
    Set the base directory for the virtual file system and add it to sys.path for imports.

    Args:
        path: New base directory path

    Returns:
        Success message
    """
    try:
        new_path = Path(path) if isinstance(path, str) else path
        new_path.mkdir(parents=True, exist_ok=True)

        # Remove old base directory from sys.path if it exists
        old_base_str = str(self.vfs.base_dir)
        if old_base_str in sys.path:
            sys.path.remove(old_base_str)

        # Update VFS base directory
        self.vfs.base_dir = new_path
        self.vfs.current_dir = new_path

        # Add new base directory to sys.path for imports
        new_base_str = str(new_path)
        if new_base_str not in sys.path:
            sys.path.insert(0, new_base_str)

        # Update user namespace paths
        self.user_ns['__path__'] = [new_base_str]

        return f"Base directory set to: {new_path} (added to sys.path)"

    except Exception as e:
        return f"Set base directory error: {str(e)}"
update_namespace(variables)

Update namespace with new variables

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
676
677
678
def update_namespace(self, variables: dict[str, Any]):
    """Update namespace with new variables"""
    self.user_ns.update(variables)
ParentNodeTransformer

Bases: NodeTransformer

Add parent references to AST nodes

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
491
492
493
494
495
496
class ParentNodeTransformer(ast.NodeTransformer):
    """Add parent references to AST nodes"""
    def visit(self, node):
        for child in ast.iter_child_nodes(node):
            child.parent = node
        return super().visit(node)
SyncReport dataclass

Report of variables synced from namespace to pipeline

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
@dataclass
class SyncReport:
    """Report of variables synced from namespace to pipeline"""
    added: dict[str, str]
    skipped: dict[str, str]  # var_name -> reason
    errors: dict[str, str]  # var_name -> error message

    def __str__(self) -> str:
        parts = []
        if self.added:
            parts.append("Added variables:")
            for name, type_ in self.added.items():
                parts.append(f"  - {name}: {type_}")
        if self.skipped:
            parts.append("\nSkipped variables:")
            for name, reason in self.skipped.items():
                parts.append(f"  - {name}: {reason}")
        if self.errors:
            parts.append("\nErrors:")
            for name, error in self.errors.items():
                parts.append(f"  - {name}: {error}")
        return "\n".join(parts)
TeeStream

Stream that writes to both console and buffer

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
475
476
477
478
479
480
481
482
483
484
485
486
487
488
class TeeStream:
    """Stream that writes to both console and buffer"""
    def __init__(self, console_stream, buffer_stream):
        self.console_stream = console_stream
        self.buffer_stream = buffer_stream

    def write(self, data):
        self.console_stream.write(data)
        self.buffer_stream.write(data)
        self.console_stream.flush()  # Ensure immediate console output

    def flush(self):
        self.console_stream.flush()
        self.buffer_stream.flush()
ToolsInterface

Minimalistic tools interface for LLMs providing code execution, virtual file system, and browser interaction capabilities.

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
class ToolsInterface:
    """
    Minimalistic tools interface for LLMs providing code execution,
    virtual file system, and browser interaction capabilities.
    """

    def __init__(self,
                 session_dir: str | None = None,
                 auto_remove: bool = True,
                 variables: dict[str, Any] | None = None,
                 variable_manager: Any | None = None):
        """
        Initialize the tools interface.

        Args:
            session_dir: Directory for session storage
            auto_remove: Whether to auto-remove temporary files
            variables: Initial variables dictionary
            variable_manager: External variable manager instance
            web_llm: LLM model for web interactions
        """
        self._session_dir = Path(session_dir) if session_dir else Path(get_app().appdata) / '.tools_sessions'
        self._session_dir.mkdir(exist_ok=True)
        self.auto_remove = auto_remove
        self.variable_manager = variable_manager

        # Initialize Python execution environment
        self.ipython = MockIPython(self._session_dir, auto_remove=auto_remove)
        if variables:
            self.ipython.user_ns.update(variables)

        # Initialize virtual file system
        self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')

        # Initialize Rust interface
        self.cargo = CargoRustInterface(self._session_dir, auto_remove=auto_remove)

        # Track execution state
        self._execution_history = []
        self._current_file = None

    async def execute_python(self, code: str) -> str:
        """
        Execute Python code in the virtual environment.

        Args:
            code: Python code to execute

        Returns:
            Execution result as string
        """
        try:
            result = await self.ipython.run_cell(code, live_output=False)

            # Update variable manager if available
            if self.variable_manager:
                for key, value in self.ipython.user_ns.items():
                    if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                        try:
                            self.variable_manager.set(f"python.{key}", value)
                        except:
                            pass  # Ignore non-serializable variables

            self._execution_history.append(('python', code, result))
            return str(result) if result else "Execution completed"

        except Exception as e:
            error_msg = f"Python execution error: {str(e)}\n{traceback.format_exc()}"
            self._execution_history.append(('python', code, error_msg))
            return error_msg

    async def execute_rust(self, code: str) -> str:
        """
        Execute Rust code using Cargo.

        Args:
            code: Rust code to execute

        Returns:
            Execution result as string
        """
        try:
            # Setup project if needed
            if not self.cargo.current_project:
                await self.cargo.setup_project("temp_rust_project")

            result = await self.cargo.run_code(code)
            self._execution_history.append(('rust', code, result))
            return result

        except Exception as e:
            error_msg = f"Rust execution error: {str(e)}"
            self._execution_history.append(('rust', code, error_msg))
            return error_msg

    async def write_file(self, filepath: str, content: str, lines: str = "") -> str:
        """
        Write content to a file in the virtual file system.

        Args:
            filepath: Path to the file
            content: Content to write
            lines: Optional line range to write (e.g., "1-3" for lines 1 to 3)

        Returns:
            Success message
        """

        try:
            if lines:
                abs_path = self.vfs.overwrite_lines(filepath, content, lines)
            else:
                abs_path = self.vfs.write_file(filepath, content)

            # Update variable manager if available
            if self.variable_manager:
                self.variable_manager.set(f"files.{filepath.replace('/', '.')}", {
                    'path': str(abs_path),
                    'size': len(content),
                    'content_preview': content[:100] + '...' if len(content) > 100 else content
                })

            return f"File written successfully: {abs_path}"

        except Exception as e:
            return f"File write error: {str(e)}"

    async def replace_in_file(self, filepath: str, old_content: str, new_content: str, precise: bool = True) -> str:
        """
        Replace exact content in file with new content.

        Args:
            filepath: Path to the file
            old_content: Exact content to replace (empty string for insertion at start)
            new_content: Content to replace with
            precise: If True, requires exact match; if False, allows single occurrence replacement

        Returns:
            Success message or error
        """
        try:
            # Read current file content
            try:
                current_content = self.vfs.read_file(filepath)
            except:
                return f"Error: File '{filepath}' not found or cannot be read"

            # Handle insertion at start (empty old_content)
            if not old_content:
                updated_content = new_content + current_content
                self.vfs.write_file(filepath, updated_content)
                return f"Content inserted at start of '{filepath}'"

            # Check if old_content exists
            if old_content not in current_content:
                return f"Error: Old content not found in '{filepath}' use read_file to check."

            # Count occurrences
            occurrences = current_content.count(old_content)

            if precise and occurrences > 1:
                return f"Error: Found {occurrences} occurrences of old content. Use precise=False to replace first occurrence."

            # Replace content (first occurrence if multiple)
            updated_content = current_content.replace(old_content, new_content, 1)

            # Write updated content
            self.vfs.write_file(filepath, updated_content)

            return f"Successfully replaced content in '{filepath}' ({occurrences} occurrence{'s' if occurrences > 1 else ''} found, 1 replaced)"

        except Exception as e:
            return f"Replace error: {str(e)}"

    async def read_file(self, filepath: str, lines: str="") -> str:
        """
        Read content from a file in the virtual file system.

        Args:
            filepath: Path to the file
            lines: Optional line range to read (e.g., "1-3" for lines 1 to 3)

        Returns:
            File content or error message
        """
        try:
            content = self.vfs.read_file(filepath)

            if lines:
                start, end = map(int, lines.split('-'))
                content = '\n'.join(content.split('\n')[start-1:end])
            # Update variable manager if available
            if self.variable_manager:
                self.variable_manager.set("files.last_read", {
                    'path': filepath,
                    'size': len(content),
                    'content_preview': content[:200] + '...' if len(content) > 200 else content
                })

            return content

        except Exception as e:
            return f"File read error: {str(e)}"

    async def list_files(self, dirpath: str = '.') -> str:
        """
        List files in a directory.

        Args:
            dirpath: Directory path to list

        Returns:
            File listing as string
        """
        try:
            files = self.vfs.list_files(dirpath)
            listing = "\n".join(f"- {file}" for file in files)
            return f"Files in '{dirpath}':\n{listing}"

        except Exception as e:
            return f"File listing error: {str(e)}"

    async def list_directory(self, dirpath: str = '.') -> str:
        """
        List contents of a directory.

        Args:
            dirpath: Directory path to list

        Returns:
            Directory listing as string
        """
        try:
            contents = self.vfs.list_directory(dirpath)
            listing = "\n".join(f"- {item}" for item in contents)

            # Update variable manager if available
            if self.variable_manager:
                self.variable_manager.set("files.last_listing", {
                    'directory': dirpath,
                    'items': contents,
                    'count': len(contents)
                })

            return f"Directory '{dirpath}' contents:\n{listing}"

        except Exception as e:
            return f"Directory listing error: {str(e)}"


    async def create_directory(self, dirpath: str) -> str:
        """
        Create a new directory.

        Args:
            dirpath: Path of directory to create

        Returns:
            Success message
        """
        try:
            abs_path = self.vfs.create_directory(dirpath)
            return f"Directory created successfully: {abs_path}"

        except Exception as e:
            return f"Directory creation error: {str(e)}"

    def set_base_directory(self, path: str) -> str:
        """
        Set the base directory for the virtual file system.

        Args:
            path: New base directory path

        Returns:
            Success message
        """
        try:
            new_path = Path(path) if isinstance(path, str) else path
            new_path = new_path.absolute()
            print(f"New path: {new_path}")
            new_path.mkdir(parents=True, exist_ok=True)
            self.vfs.base_dir = new_path
            self.vfs.current_dir = new_path

            # Update MockIPython base directory and sys.path
            result = self.ipython.set_base_directory(path)

            return result

        except Exception as e:
            return f"Set base directory error: {str(e)}"

    async def set_current_file(self, filepath: str) -> str:
        """
        Set the current file for Python execution context.

        Args:
            filepath: Path to set as current file

        Returns:
            Success message
        """
        try:
            abs_path = self.vfs._resolve_path(filepath)
            self.ipython.user_ns['__file__'] = str(abs_path)
            self._current_file = str(abs_path)

            return f"Current file set to: {abs_path}"

        except Exception as e:
            return f"Set current file error: {str(e)}"

    async def install_package(self, package_name: str, version: str | None = None) -> str:
        """
        Install a Python package in the virtual environment.

        Args:
            package_name: Name of the package to install
            version: Optional specific version to install

        Returns:
            Installation result
        """
        try:
            code = f"""
auto_install('{package_name}'{f", version='{version}'" if version else ""})
import {package_name.split('[')[0]}  # Import base package name
print(f"Successfully imported {package_name}")
"""
            result = await self.execute_python(code)
            return result

        except Exception as e:
            return f"Package installation error: {str(e)}"

    async def get_execution_history(self) -> str:
        """
        Get the execution history.

        Returns:
            Execution history as formatted string
        """
        if not self._execution_history:
            return "No execution history available."

        history_lines = []
        for i, (lang, code, result) in enumerate(self._execution_history[-10:], 1):
            history_lines.append(f"[{i}] {lang.upper()}:")
            history_lines.append(f"    Code: {code[:100]}..." if len(code) > 100 else f"    Code: {code}")
            history_lines.append(
                f"    Result: {str(result)[:200]}..." if len(str(result)) > 200 else f"    Result: {result}")
            history_lines.append("")

        return "\n".join(history_lines)

    async def clear_session(self) -> str:
        """
        Clear the current session (variables, history, files).

        Returns:
            Success message
        """
        try:
            # Reset Python environment
            self.ipython.reset()

            # Clear execution history
            self._execution_history.clear()

            # Clear VFS if auto_remove is enabled
            if self.auto_remove:
                shutil.rmtree(self.vfs.base_dir, ignore_errors=True)
                self.vfs.base_dir.mkdir(parents=True, exist_ok=True)
                self.vfs.virtual_files.clear()

            # Reset current file
            self._current_file = None

            return "Session cleared successfully"

        except Exception as e:
            return f"Clear session error: {str(e)}"

    async def get_variables(self) -> str:
        """
        Get current variables in JSON format.

        Returns:
            Variables as JSON string
        """
        try:
            # Get Python variables
            py_vars = {}
            for key, value in self.ipython.user_ns.items():
                if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                    try:
                        # Try to serialize the value
                        json.dumps(value, default=str)
                        py_vars[key] = str(value)[:200] if len(str(value)) > 200 else value
                    except:
                        py_vars[key] = f"<{type(value).__name__}>"

            result = {
                'python_variables': py_vars,
                'current_file': self._current_file,
                'vfs_base': str(self.vfs.base_dir),
                'execution_count': len(self._execution_history)
            }

            return json.dumps(result, indent=2, default=str)

        except Exception as e:
            return f"Get variables error: {str(e)}"

    def get_tools(self, name:str=None) -> list[tuple[Any, str, str]]:
        """
        Get all available tools as list of tuples (function, name, description).

        Returns:
            List of tool tuples
        """
        tools = [
            # Code execution tools
            (self.execute_python, "execute_python",
             "Execute Python code in virtual environment. all variables ar available under the python scope.\n"
             "The isaa_instance is available as isaa_instance in the python code."
             " Args: code (str) -> str"),

            # (self.execute_rust, "execute_rust",
            #  "Execute Rust code using Cargo. Args: code (str) -> str"),

            # File system tools
            (self.write_file, "write_file",
             "Write content to file in virtual filesystem. lines is a string with the line range to write (e.g., '1-3' for lines 1 to 3) Args: filepath (str), content (str), lines (str) = '' -> str"),

            (self.write_file, "create_file",
             "Write content to file in virtual filesystem.  Args: filepath (str), content (str) -> str"),

            (self.replace_in_file, "replace_in_file",
             "Replace exact content in file. Args: filepath (str), old_content (str), new_content (str), precise (bool) = True -> str"),

            (self.read_file, "read_file",
             "Read content from file in virtual filesystem. lines is a string with the line range to read (e.g., '1-3' for lines 1 to 3) Args: filepath (str), lines (str) = '' -> str"),

            (self.list_files, "list_files",
             "List files in directory. Args: dirpath (str) = '.' -> str"),

            (self.list_directory, "list_directory",
             "List directory contents. Args: dirpath (str) = '.' -> str"),

            (self.create_directory, "create_directory",
             "Create new directory. Args: dirpath (str) -> str"),

            # Configuration tools
            (self.set_base_directory, "set_base_directory",
             "Set base directory for virtual filesystem. Args: path (str) -> str"),

            (self.set_current_file, "set_current_file",
             "Set current file for Python execution context. Args: filepath (str) -> str"),

            (self.install_package, "install_package",
             "Install Python package. Args: package_name (str), version (Optional[str]) -> str"),

            # Session management tools
            (self.get_execution_history, "get_execution_history",
             "Get execution history. Args: None -> str"),

            (self.clear_session, "clear_session",
             "Clear current session. Args: None -> str"),

            (self.get_variables, "get_variables",
             "Get current variables as JSON. Args: None -> str"),
        ]
        if name is not None:
            tools = [t for t in tools if t[1] == name][0]
        return tools

    def __aenter__(self):
        return self

    async def __aexit__(self, *exe):
        await asyncio.sleep(0.01)
__init__(session_dir=None, auto_remove=True, variables=None, variable_manager=None)

Initialize the tools interface.

Parameters:

Name Type Description Default
session_dir str | None

Directory for session storage

None
auto_remove bool

Whether to auto-remove temporary files

True
variables dict[str, Any] | None

Initial variables dictionary

None
variable_manager Any | None

External variable manager instance

None
web_llm

LLM model for web interactions

required
Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
def __init__(self,
             session_dir: str | None = None,
             auto_remove: bool = True,
             variables: dict[str, Any] | None = None,
             variable_manager: Any | None = None):
    """
    Initialize the tools interface.

    Args:
        session_dir: Directory for session storage
        auto_remove: Whether to auto-remove temporary files
        variables: Initial variables dictionary
        variable_manager: External variable manager instance
        web_llm: LLM model for web interactions
    """
    self._session_dir = Path(session_dir) if session_dir else Path(get_app().appdata) / '.tools_sessions'
    self._session_dir.mkdir(exist_ok=True)
    self.auto_remove = auto_remove
    self.variable_manager = variable_manager

    # Initialize Python execution environment
    self.ipython = MockIPython(self._session_dir, auto_remove=auto_remove)
    if variables:
        self.ipython.user_ns.update(variables)

    # Initialize virtual file system
    self.vfs = VirtualFileSystem(self._session_dir / 'virtual_fs')

    # Initialize Rust interface
    self.cargo = CargoRustInterface(self._session_dir, auto_remove=auto_remove)

    # Track execution state
    self._execution_history = []
    self._current_file = None
clear_session() async

Clear the current session (variables, history, files).

Returns:

Type Description
str

Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
async def clear_session(self) -> str:
    """
    Clear the current session (variables, history, files).

    Returns:
        Success message
    """
    try:
        # Reset Python environment
        self.ipython.reset()

        # Clear execution history
        self._execution_history.clear()

        # Clear VFS if auto_remove is enabled
        if self.auto_remove:
            shutil.rmtree(self.vfs.base_dir, ignore_errors=True)
            self.vfs.base_dir.mkdir(parents=True, exist_ok=True)
            self.vfs.virtual_files.clear()

        # Reset current file
        self._current_file = None

        return "Session cleared successfully"

    except Exception as e:
        return f"Clear session error: {str(e)}"
create_directory(dirpath) async

Create a new directory.

Parameters:

Name Type Description Default
dirpath str

Path of directory to create

required

Returns:

Type Description
str

Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
async def create_directory(self, dirpath: str) -> str:
    """
    Create a new directory.

    Args:
        dirpath: Path of directory to create

    Returns:
        Success message
    """
    try:
        abs_path = self.vfs.create_directory(dirpath)
        return f"Directory created successfully: {abs_path}"

    except Exception as e:
        return f"Directory creation error: {str(e)}"
execute_python(code) async

Execute Python code in the virtual environment.

Parameters:

Name Type Description Default
code str

Python code to execute

required

Returns:

Type Description
str

Execution result as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
async def execute_python(self, code: str) -> str:
    """
    Execute Python code in the virtual environment.

    Args:
        code: Python code to execute

    Returns:
        Execution result as string
    """
    try:
        result = await self.ipython.run_cell(code, live_output=False)

        # Update variable manager if available
        if self.variable_manager:
            for key, value in self.ipython.user_ns.items():
                if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                    try:
                        self.variable_manager.set(f"python.{key}", value)
                    except:
                        pass  # Ignore non-serializable variables

        self._execution_history.append(('python', code, result))
        return str(result) if result else "Execution completed"

    except Exception as e:
        error_msg = f"Python execution error: {str(e)}\n{traceback.format_exc()}"
        self._execution_history.append(('python', code, error_msg))
        return error_msg
execute_rust(code) async

Execute Rust code using Cargo.

Parameters:

Name Type Description Default
code str

Rust code to execute

required

Returns:

Type Description
str

Execution result as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
async def execute_rust(self, code: str) -> str:
    """
    Execute Rust code using Cargo.

    Args:
        code: Rust code to execute

    Returns:
        Execution result as string
    """
    try:
        # Setup project if needed
        if not self.cargo.current_project:
            await self.cargo.setup_project("temp_rust_project")

        result = await self.cargo.run_code(code)
        self._execution_history.append(('rust', code, result))
        return result

    except Exception as e:
        error_msg = f"Rust execution error: {str(e)}"
        self._execution_history.append(('rust', code, error_msg))
        return error_msg
get_execution_history() async

Get the execution history.

Returns:

Type Description
str

Execution history as formatted string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
async def get_execution_history(self) -> str:
    """
    Get the execution history.

    Returns:
        Execution history as formatted string
    """
    if not self._execution_history:
        return "No execution history available."

    history_lines = []
    for i, (lang, code, result) in enumerate(self._execution_history[-10:], 1):
        history_lines.append(f"[{i}] {lang.upper()}:")
        history_lines.append(f"    Code: {code[:100]}..." if len(code) > 100 else f"    Code: {code}")
        history_lines.append(
            f"    Result: {str(result)[:200]}..." if len(str(result)) > 200 else f"    Result: {result}")
        history_lines.append("")

    return "\n".join(history_lines)
get_tools(name=None)

Get all available tools as list of tuples (function, name, description).

Returns:

Type Description
list[tuple[Any, str, str]]

List of tool tuples

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
def get_tools(self, name:str=None) -> list[tuple[Any, str, str]]:
    """
    Get all available tools as list of tuples (function, name, description).

    Returns:
        List of tool tuples
    """
    tools = [
        # Code execution tools
        (self.execute_python, "execute_python",
         "Execute Python code in virtual environment. all variables ar available under the python scope.\n"
         "The isaa_instance is available as isaa_instance in the python code."
         " Args: code (str) -> str"),

        # (self.execute_rust, "execute_rust",
        #  "Execute Rust code using Cargo. Args: code (str) -> str"),

        # File system tools
        (self.write_file, "write_file",
         "Write content to file in virtual filesystem. lines is a string with the line range to write (e.g., '1-3' for lines 1 to 3) Args: filepath (str), content (str), lines (str) = '' -> str"),

        (self.write_file, "create_file",
         "Write content to file in virtual filesystem.  Args: filepath (str), content (str) -> str"),

        (self.replace_in_file, "replace_in_file",
         "Replace exact content in file. Args: filepath (str), old_content (str), new_content (str), precise (bool) = True -> str"),

        (self.read_file, "read_file",
         "Read content from file in virtual filesystem. lines is a string with the line range to read (e.g., '1-3' for lines 1 to 3) Args: filepath (str), lines (str) = '' -> str"),

        (self.list_files, "list_files",
         "List files in directory. Args: dirpath (str) = '.' -> str"),

        (self.list_directory, "list_directory",
         "List directory contents. Args: dirpath (str) = '.' -> str"),

        (self.create_directory, "create_directory",
         "Create new directory. Args: dirpath (str) -> str"),

        # Configuration tools
        (self.set_base_directory, "set_base_directory",
         "Set base directory for virtual filesystem. Args: path (str) -> str"),

        (self.set_current_file, "set_current_file",
         "Set current file for Python execution context. Args: filepath (str) -> str"),

        (self.install_package, "install_package",
         "Install Python package. Args: package_name (str), version (Optional[str]) -> str"),

        # Session management tools
        (self.get_execution_history, "get_execution_history",
         "Get execution history. Args: None -> str"),

        (self.clear_session, "clear_session",
         "Clear current session. Args: None -> str"),

        (self.get_variables, "get_variables",
         "Get current variables as JSON. Args: None -> str"),
    ]
    if name is not None:
        tools = [t for t in tools if t[1] == name][0]
    return tools
get_variables() async

Get current variables in JSON format.

Returns:

Type Description
str

Variables as JSON string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
async def get_variables(self) -> str:
    """
    Get current variables in JSON format.

    Returns:
        Variables as JSON string
    """
    try:
        # Get Python variables
        py_vars = {}
        for key, value in self.ipython.user_ns.items():
            if not key.startswith('_') and key not in ['__name__', '__builtins__']:
                try:
                    # Try to serialize the value
                    json.dumps(value, default=str)
                    py_vars[key] = str(value)[:200] if len(str(value)) > 200 else value
                except:
                    py_vars[key] = f"<{type(value).__name__}>"

        result = {
            'python_variables': py_vars,
            'current_file': self._current_file,
            'vfs_base': str(self.vfs.base_dir),
            'execution_count': len(self._execution_history)
        }

        return json.dumps(result, indent=2, default=str)

    except Exception as e:
        return f"Get variables error: {str(e)}"
install_package(package_name, version=None) async

Install a Python package in the virtual environment.

Parameters:

Name Type Description Default
package_name str

Name of the package to install

required
version str | None

Optional specific version to install

None

Returns:

Type Description
str

Installation result

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
    async def install_package(self, package_name: str, version: str | None = None) -> str:
        """
        Install a Python package in the virtual environment.

        Args:
            package_name: Name of the package to install
            version: Optional specific version to install

        Returns:
            Installation result
        """
        try:
            code = f"""
auto_install('{package_name}'{f", version='{version}'" if version else ""})
import {package_name.split('[')[0]}  # Import base package name
print(f"Successfully imported {package_name}")
"""
            result = await self.execute_python(code)
            return result

        except Exception as e:
            return f"Package installation error: {str(e)}"
list_directory(dirpath='.') async

List contents of a directory.

Parameters:

Name Type Description Default
dirpath str

Directory path to list

'.'

Returns:

Type Description
str

Directory listing as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
async def list_directory(self, dirpath: str = '.') -> str:
    """
    List contents of a directory.

    Args:
        dirpath: Directory path to list

    Returns:
        Directory listing as string
    """
    try:
        contents = self.vfs.list_directory(dirpath)
        listing = "\n".join(f"- {item}" for item in contents)

        # Update variable manager if available
        if self.variable_manager:
            self.variable_manager.set("files.last_listing", {
                'directory': dirpath,
                'items': contents,
                'count': len(contents)
            })

        return f"Directory '{dirpath}' contents:\n{listing}"

    except Exception as e:
        return f"Directory listing error: {str(e)}"
list_files(dirpath='.') async

List files in a directory.

Parameters:

Name Type Description Default
dirpath str

Directory path to list

'.'

Returns:

Type Description
str

File listing as string

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
async def list_files(self, dirpath: str = '.') -> str:
    """
    List files in a directory.

    Args:
        dirpath: Directory path to list

    Returns:
        File listing as string
    """
    try:
        files = self.vfs.list_files(dirpath)
        listing = "\n".join(f"- {file}" for file in files)
        return f"Files in '{dirpath}':\n{listing}"

    except Exception as e:
        return f"File listing error: {str(e)}"
read_file(filepath, lines='') async

Read content from a file in the virtual file system.

Parameters:

Name Type Description Default
filepath str

Path to the file

required
lines str

Optional line range to read (e.g., "1-3" for lines 1 to 3)

''

Returns:

Type Description
str

File content or error message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
async def read_file(self, filepath: str, lines: str="") -> str:
    """
    Read content from a file in the virtual file system.

    Args:
        filepath: Path to the file
        lines: Optional line range to read (e.g., "1-3" for lines 1 to 3)

    Returns:
        File content or error message
    """
    try:
        content = self.vfs.read_file(filepath)

        if lines:
            start, end = map(int, lines.split('-'))
            content = '\n'.join(content.split('\n')[start-1:end])
        # Update variable manager if available
        if self.variable_manager:
            self.variable_manager.set("files.last_read", {
                'path': filepath,
                'size': len(content),
                'content_preview': content[:200] + '...' if len(content) > 200 else content
            })

        return content

    except Exception as e:
        return f"File read error: {str(e)}"
replace_in_file(filepath, old_content, new_content, precise=True) async

Replace exact content in file with new content.

Parameters:

Name Type Description Default
filepath str

Path to the file

required
old_content str

Exact content to replace (empty string for insertion at start)

required
new_content str

Content to replace with

required
precise bool

If True, requires exact match; if False, allows single occurrence replacement

True

Returns:

Type Description
str

Success message or error

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
async def replace_in_file(self, filepath: str, old_content: str, new_content: str, precise: bool = True) -> str:
    """
    Replace exact content in file with new content.

    Args:
        filepath: Path to the file
        old_content: Exact content to replace (empty string for insertion at start)
        new_content: Content to replace with
        precise: If True, requires exact match; if False, allows single occurrence replacement

    Returns:
        Success message or error
    """
    try:
        # Read current file content
        try:
            current_content = self.vfs.read_file(filepath)
        except:
            return f"Error: File '{filepath}' not found or cannot be read"

        # Handle insertion at start (empty old_content)
        if not old_content:
            updated_content = new_content + current_content
            self.vfs.write_file(filepath, updated_content)
            return f"Content inserted at start of '{filepath}'"

        # Check if old_content exists
        if old_content not in current_content:
            return f"Error: Old content not found in '{filepath}' use read_file to check."

        # Count occurrences
        occurrences = current_content.count(old_content)

        if precise and occurrences > 1:
            return f"Error: Found {occurrences} occurrences of old content. Use precise=False to replace first occurrence."

        # Replace content (first occurrence if multiple)
        updated_content = current_content.replace(old_content, new_content, 1)

        # Write updated content
        self.vfs.write_file(filepath, updated_content)

        return f"Successfully replaced content in '{filepath}' ({occurrences} occurrence{'s' if occurrences > 1 else ''} found, 1 replaced)"

    except Exception as e:
        return f"Replace error: {str(e)}"
set_base_directory(path)

Set the base directory for the virtual file system.

Parameters:

Name Type Description Default
path str

New base directory path

required

Returns:

Type Description
str

Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
def set_base_directory(self, path: str) -> str:
    """
    Set the base directory for the virtual file system.

    Args:
        path: New base directory path

    Returns:
        Success message
    """
    try:
        new_path = Path(path) if isinstance(path, str) else path
        new_path = new_path.absolute()
        print(f"New path: {new_path}")
        new_path.mkdir(parents=True, exist_ok=True)
        self.vfs.base_dir = new_path
        self.vfs.current_dir = new_path

        # Update MockIPython base directory and sys.path
        result = self.ipython.set_base_directory(path)

        return result

    except Exception as e:
        return f"Set base directory error: {str(e)}"
set_current_file(filepath) async

Set the current file for Python execution context.

Parameters:

Name Type Description Default
filepath str

Path to set as current file

required

Returns:

Type Description
str

Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
async def set_current_file(self, filepath: str) -> str:
    """
    Set the current file for Python execution context.

    Args:
        filepath: Path to set as current file

    Returns:
        Success message
    """
    try:
        abs_path = self.vfs._resolve_path(filepath)
        self.ipython.user_ns['__file__'] = str(abs_path)
        self._current_file = str(abs_path)

        return f"Current file set to: {abs_path}"

    except Exception as e:
        return f"Set current file error: {str(e)}"
write_file(filepath, content, lines='') async

Write content to a file in the virtual file system.

Parameters:

Name Type Description Default
filepath str

Path to the file

required
content str

Content to write

required
lines str

Optional line range to write (e.g., "1-3" for lines 1 to 3)

''

Returns:

Type Description
str

Success message

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
async def write_file(self, filepath: str, content: str, lines: str = "") -> str:
    """
    Write content to a file in the virtual file system.

    Args:
        filepath: Path to the file
        content: Content to write
        lines: Optional line range to write (e.g., "1-3" for lines 1 to 3)

    Returns:
        Success message
    """

    try:
        if lines:
            abs_path = self.vfs.overwrite_lines(filepath, content, lines)
        else:
            abs_path = self.vfs.write_file(filepath, content)

        # Update variable manager if available
        if self.variable_manager:
            self.variable_manager.set(f"files.{filepath.replace('/', '.')}", {
                'path': str(abs_path),
                'size': len(content),
                'content_preview': content[:100] + '...' if len(content) > 100 else content
            })

        return f"File written successfully: {abs_path}"

    except Exception as e:
        return f"File write error: {str(e)}"
VirtualEnvContext

Context manager for temporary virtual environment activation

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
class VirtualEnvContext:
    """Context manager for temporary virtual environment activation"""

    def __init__(self, venv_path: Path):
        self.venv_path = venv_path
        self._original_path = None
        self._original_sys_path = None
        self._original_prefix = None
        self._original_virtual_env = None

    def _get_venv_paths(self):
        """Get virtual environment paths based on platform"""
        if sys.platform == 'win32':
            site_packages = self.venv_path / 'Lib' / 'site-packages'
            scripts_dir = self.venv_path / 'Scripts'
            python_path = scripts_dir / 'python.exe'
        else:
            python_version = f'python{sys.version_info.major}.{sys.version_info.minor}'
            site_packages = self.venv_path / 'lib' / python_version / 'site-packages'
            scripts_dir = self.venv_path / 'bin'
            python_path = scripts_dir / 'python'

        return site_packages, scripts_dir, python_path

    def __enter__(self):
        # Save original state
        self._original_path = os.environ.get('PATH', '')
        self._original_sys_path = sys.path.copy()
        self._original_prefix = sys.prefix
        self._original_virtual_env = os.environ.get('VIRTUAL_ENV')

        # Get venv paths
        site_packages, scripts_dir, python_path = self._get_venv_paths()

        # Modify environment for venv
        if scripts_dir.exists():
            new_path = os.pathsep.join([str(scripts_dir), self._original_path])
            os.environ['PATH'] = new_path

        if site_packages.exists():
            sys.path.insert(0, str(site_packages))

        os.environ['VIRTUAL_ENV'] = str(self.venv_path)

        # Return the python executable path for potential subprocess calls
        return str(python_path)

    def __exit__(self, exc_type, exc_val, exc_tb):
        # Restore original state
        os.environ['PATH'] = self._original_path
        sys.path = self._original_sys_path

        if self._original_virtual_env is None:
            os.environ.pop('VIRTUAL_ENV', None)
        else:
            os.environ['VIRTUAL_ENV'] = self._original_virtual_env
VirtualFileSystem
Source code in toolboxv2/mods/isaa/CodingAgent/live.py
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
class VirtualFileSystem:
    def __init__(self, base_dir: Path):
        self.base_dir = base_dir
        self.current_dir = base_dir
        self.virtual_files: dict[str, str] = {}
        self.base_dir.mkdir(parents=True, exist_ok=True)

    def ifile_exists(self, filepath: str | Path) -> bool:
        """Check if a file exists"""
        abs_path = self._resolve_path(filepath)
        return abs_path.exists()

    def overwrite_lines(self, filepath: str | Path, new_content: str, lines: str = "") -> str:
        """Overwrite lines in a file"""
        try:
            content = self.read_file(filepath)
            content_lines = content.split('\n')
            start, end = map(int, lines.split('-'))
            # overwrite specif lines with new content keep the rest
            content_before = content_lines[:start-1]
            content_after = content_lines[end:]
            content_lines = content_before + new_content.split('\n') + content_after
            content = '\n'.join(content_lines)
            return self.write_file(filepath, content)
        except Exception as e:
            return f"Overwrite lines failed: {str(e)}"

    def write_file(self, filepath: str | Path, content: str) -> Path:
        """Write content to a virtual file and persist to disk using UTF-8"""
        try:
            abs_path = self._resolve_path(filepath)
        except ValueError:
            print("invalid :", filepath)
            filepath = "src/temp/_temp_fix.py"
            abs_path = self._resolve_path(filepath)
        abs_path.parent.mkdir(parents=True, exist_ok=True)

        # Store in virtual filesystem
        rel_path = str(abs_path.relative_to(self.base_dir))
        self.virtual_files[rel_path] = content

        # Write to actual filesystem with UTF-8 encoding
        with open(abs_path, 'w', encoding='utf-8', errors='replace') as f:
            f.write(content)

        parent_dir_str = str(abs_path.parent.absolute())
        if parent_dir_str not in sys.path and abs_path.suffix == '.py':
            sys.path.insert(0, parent_dir_str)

        return abs_path

    def read_file(self, filepath: str | Path) -> str:
        """Read content from a virtual file using UTF-8"""
        abs_path = self._resolve_path(filepath)
        if not abs_path.exists():
            raise FileNotFoundError(f"File not found: {filepath}")

        rel_path = str(abs_path.relative_to(self.base_dir))

        # Check virtual filesystem first
        if rel_path in self.virtual_files:
            return self.virtual_files[rel_path]

        # Fall back to reading from disk with UTF-8 encoding
        with open(abs_path, encoding='utf-8', errors='replace') as f:
            content = f.read()
            self.virtual_files[rel_path] = content
            return content

    def delete_file(self, filepath: str | Path):
        """Delete a virtual file"""
        abs_path = self._resolve_path(filepath)
        rel_path = str(abs_path.relative_to(self.base_dir))

        if rel_path in self.virtual_files:
            del self.virtual_files[rel_path]

        if abs_path.exists():
            abs_path.unlink()

    def create_directory(self, dirpath: str | Path):
        """Create a new directory"""
        abs_path = self._resolve_path(dirpath)
        abs_path.mkdir(parents=True, exist_ok=True)
        return abs_path

    def list_files(self, dirpath: str | Path = '.') -> list:
        """List files in a directory"""
        abs_path = self._resolve_path(dirpath)
        if not abs_path.exists():
            raise FileNotFoundError(f"Directory not found: {dirpath}")
        return [p.name for p in abs_path.iterdir() if p.is_file()]

    def list_directory(self, dirpath: str | Path = '.') -> list:
        """List contents of a directory"""
        abs_path = self._resolve_path(dirpath)
        if not abs_path.exists():
            raise FileNotFoundError(f"Directory not found: {dirpath}")
        return [p.name for p in abs_path.iterdir()]

    def change_directory(self, dirpath: str | Path):
        """Change current working directory"""
        new_dir = self._resolve_path(dirpath)
        if not new_dir.exists() or not new_dir.is_dir():
            raise NotADirectoryError(f"Directory not found: {dirpath}")
        self.current_dir = new_dir

    def _resolve_path(self, filepath: str | Path) -> Path:
        """Convert relative path to absolute path"""
        filepath = Path(filepath)
        if filepath.is_absolute():
            if not str(filepath).startswith(str(self.base_dir)):
                raise ValueError("Path must be within base directory")
            return filepath
        return (self.current_dir / filepath).resolve()

    def save_state(self, state_file: Path):
        """Save virtual filesystem state to disk"""
        state = {
            'current_dir': str(self.current_dir.relative_to(self.base_dir)),
            'virtual_files': self.virtual_files
        }
        with open(state_file, 'w') as f:
            json.dump(state, f)

    def load_state(self, state_file: Path):
        """Load virtual filesystem state from disk"""
        if not state_file.exists():
            return

        with open(state_file) as f:
            state = json.load(f)
            self.current_dir = self.base_dir / state['current_dir']
            self.virtual_files = state['virtual_files']

    def print_file_structure(self, start_path: str | Path = '.', indent: str = ''):
        """Print the file structure starting from the given path"""
        start_path = self._resolve_path(start_path)
        if not start_path.exists():
            s = f"Path not found: {start_path}"
            return s

        s = f"{indent}{start_path.name}/"
        for item in sorted(start_path.iterdir()):
            if item.is_dir():
               s+= self.print_file_structure(item, indent + '  ')
            else:
                s = f"{indent}  {item.name}"
        return s
change_directory(dirpath)

Change current working directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
367
368
369
370
371
372
def change_directory(self, dirpath: str | Path):
    """Change current working directory"""
    new_dir = self._resolve_path(dirpath)
    if not new_dir.exists() or not new_dir.is_dir():
        raise NotADirectoryError(f"Directory not found: {dirpath}")
    self.current_dir = new_dir
create_directory(dirpath)

Create a new directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
347
348
349
350
351
def create_directory(self, dirpath: str | Path):
    """Create a new directory"""
    abs_path = self._resolve_path(dirpath)
    abs_path.mkdir(parents=True, exist_ok=True)
    return abs_path
delete_file(filepath)

Delete a virtual file

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
336
337
338
339
340
341
342
343
344
345
def delete_file(self, filepath: str | Path):
    """Delete a virtual file"""
    abs_path = self._resolve_path(filepath)
    rel_path = str(abs_path.relative_to(self.base_dir))

    if rel_path in self.virtual_files:
        del self.virtual_files[rel_path]

    if abs_path.exists():
        abs_path.unlink()
ifile_exists(filepath)

Check if a file exists

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
274
275
276
277
def ifile_exists(self, filepath: str | Path) -> bool:
    """Check if a file exists"""
    abs_path = self._resolve_path(filepath)
    return abs_path.exists()
list_directory(dirpath='.')

List contents of a directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
360
361
362
363
364
365
def list_directory(self, dirpath: str | Path = '.') -> list:
    """List contents of a directory"""
    abs_path = self._resolve_path(dirpath)
    if not abs_path.exists():
        raise FileNotFoundError(f"Directory not found: {dirpath}")
    return [p.name for p in abs_path.iterdir()]
list_files(dirpath='.')

List files in a directory

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
353
354
355
356
357
358
def list_files(self, dirpath: str | Path = '.') -> list:
    """List files in a directory"""
    abs_path = self._resolve_path(dirpath)
    if not abs_path.exists():
        raise FileNotFoundError(f"Directory not found: {dirpath}")
    return [p.name for p in abs_path.iterdir() if p.is_file()]
load_state(state_file)

Load virtual filesystem state from disk

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
392
393
394
395
396
397
398
399
400
def load_state(self, state_file: Path):
    """Load virtual filesystem state from disk"""
    if not state_file.exists():
        return

    with open(state_file) as f:
        state = json.load(f)
        self.current_dir = self.base_dir / state['current_dir']
        self.virtual_files = state['virtual_files']
overwrite_lines(filepath, new_content, lines='')

Overwrite lines in a file

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
279
280
281
282
283
284
285
286
287
288
289
290
291
292
def overwrite_lines(self, filepath: str | Path, new_content: str, lines: str = "") -> str:
    """Overwrite lines in a file"""
    try:
        content = self.read_file(filepath)
        content_lines = content.split('\n')
        start, end = map(int, lines.split('-'))
        # overwrite specif lines with new content keep the rest
        content_before = content_lines[:start-1]
        content_after = content_lines[end:]
        content_lines = content_before + new_content.split('\n') + content_after
        content = '\n'.join(content_lines)
        return self.write_file(filepath, content)
    except Exception as e:
        return f"Overwrite lines failed: {str(e)}"
print_file_structure(start_path='.', indent='')

Print the file structure starting from the given path

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
402
403
404
405
406
407
408
409
410
411
412
413
414
415
def print_file_structure(self, start_path: str | Path = '.', indent: str = ''):
    """Print the file structure starting from the given path"""
    start_path = self._resolve_path(start_path)
    if not start_path.exists():
        s = f"Path not found: {start_path}"
        return s

    s = f"{indent}{start_path.name}/"
    for item in sorted(start_path.iterdir()):
        if item.is_dir():
           s+= self.print_file_structure(item, indent + '  ')
        else:
            s = f"{indent}  {item.name}"
    return s
read_file(filepath)

Read content from a virtual file using UTF-8

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
def read_file(self, filepath: str | Path) -> str:
    """Read content from a virtual file using UTF-8"""
    abs_path = self._resolve_path(filepath)
    if not abs_path.exists():
        raise FileNotFoundError(f"File not found: {filepath}")

    rel_path = str(abs_path.relative_to(self.base_dir))

    # Check virtual filesystem first
    if rel_path in self.virtual_files:
        return self.virtual_files[rel_path]

    # Fall back to reading from disk with UTF-8 encoding
    with open(abs_path, encoding='utf-8', errors='replace') as f:
        content = f.read()
        self.virtual_files[rel_path] = content
        return content
save_state(state_file)

Save virtual filesystem state to disk

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
383
384
385
386
387
388
389
390
def save_state(self, state_file: Path):
    """Save virtual filesystem state to disk"""
    state = {
        'current_dir': str(self.current_dir.relative_to(self.base_dir)),
        'virtual_files': self.virtual_files
    }
    with open(state_file, 'w') as f:
        json.dump(state, f)
write_file(filepath, content)

Write content to a virtual file and persist to disk using UTF-8

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
def write_file(self, filepath: str | Path, content: str) -> Path:
    """Write content to a virtual file and persist to disk using UTF-8"""
    try:
        abs_path = self._resolve_path(filepath)
    except ValueError:
        print("invalid :", filepath)
        filepath = "src/temp/_temp_fix.py"
        abs_path = self._resolve_path(filepath)
    abs_path.parent.mkdir(parents=True, exist_ok=True)

    # Store in virtual filesystem
    rel_path = str(abs_path.relative_to(self.base_dir))
    self.virtual_files[rel_path] = content

    # Write to actual filesystem with UTF-8 encoding
    with open(abs_path, 'w', encoding='utf-8', errors='replace') as f:
        f.write(content)

    parent_dir_str = str(abs_path.parent.absolute())
    if parent_dir_str not in sys.path and abs_path.suffix == '.py':
        sys.path.insert(0, parent_dir_str)

    return abs_path
auto_install(package_name, install_method='pip', upgrade=False, quiet=False, version=None, extra_args=None)

Enhanced auto-save import with version and extra arguments support

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
def auto_install(package_name, install_method='pip', upgrade=False, quiet=False, version=None, extra_args=None):
    '''
    Enhanced auto-save import with version and extra arguments support
    '''
    try:
        # Attempt to import the package
        return importlib.import_module(package_name)
    except ImportError:
        # Package not found, prepare for installation
        print(f"Package '{package_name}' not found. Attempting to install...")
        try:
            # Determine Python executable based on virtual environment
            venv_path = os.environ.get('VIRTUAL_ENV')
            if venv_path:
                venv_path = Path(venv_path)
                if sys.platform == 'win32':
                    python_exec = str(venv_path / 'Scripts' / 'python.exe')
                else:
                    python_exec = str(venv_path / 'bin' / 'python')
                # Check if the Python executable exists
                if not Path(python_exec).exists():
                    python_exec = sys.executable
            else:
                python_exec = sys.executable

            # Construct installation command with more flexibility
            install_cmd = [python_exec, "-m", install_method, "install"]
            if upgrade:
                install_cmd.append("--upgrade")
            # Support specific version installation
            if version:
                install_cmd.append(f"{package_name}=={version}")
            else:
                install_cmd.append(package_name)
            # Add extra arguments if provided
            if extra_args:
                install_cmd.extend(extra_args)
            # Run installation with appropriate verbosity
            installation_output = subprocess.run(
                install_cmd,
                capture_output=quiet,
                text=True
            )
            # Check installation status
            if installation_output.returncode == 0:
                print(f"Successfully installed {package_name}")
                return importlib.import_module(package_name)
            else:
                raise Exception(f"Installation failed: {installation_output.stderr}")
        except Exception as install_error:
            print(f"Error installing {package_name}: {install_error}")
            return None
sync_globals_to_vars(pipeline, namespace=None, prefix=None, include_types=None, exclude_patterns=None, exclude_private=True, deep_copy=False, only_serializable=False)
Sync global variables or a specific namespace to pipeline variables.

Args:
    pipeline: Pipeline instance to sync variables to
    namespace: Optional dictionary of variables (defaults to globals())
    prefix: Optional prefix for variable names (e.g., 'global_')
    include_types: Only include variables of these types
    exclude_patterns: List of regex patterns to exclude
    exclude_private: Exclude variables starting with underscore
    deep_copy: Create deep copies of variables instead of references
    only_serializable: Only include variables that can be serialized

Returns:
    SyncReport with details about added, skipped and error variables

Usage example:
Basic usage - sync all globals

report = sync_globals_to_vars(pipeline)

Sync only numeric types with prefix

report = sync_globals_to_vars( pipeline, include_types=[int, float], prefix="global_" )

Sync from specific namespace

import numpy as np namespace = {"arr": np.array([1,2,3])} report = sync_globals_to_vars(pipeline, namespace=namespace)

Sync with deep copy and serialization check

report = sync_globals_to_vars( pipeline, deep_copy=True, only_serializable=True )

Source code in toolboxv2/mods/isaa/CodingAgent/live.py
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
def sync_globals_to_vars(
    pipeline: Any,
    namespace: dict[str, Any] | None = None,
    prefix: str | None = None,
    include_types: type | list[type] | None = None,
    exclude_patterns: list[str] | None = None,
    exclude_private: bool = True,
    deep_copy: bool = False,
    only_serializable: bool = False
) -> SyncReport:
    """
    Sync global variables or a specific namespace to pipeline variables.

    Args:
        pipeline: Pipeline instance to sync variables to
        namespace: Optional dictionary of variables (defaults to globals())
        prefix: Optional prefix for variable names (e.g., 'global_')
        include_types: Only include variables of these types
        exclude_patterns: List of regex patterns to exclude
        exclude_private: Exclude variables starting with underscore
        deep_copy: Create deep copies of variables instead of references
        only_serializable: Only include variables that can be serialized

    Returns:
        SyncReport with details about added, skipped and error variables

    Usage example:
# Basic usage - sync all globals
report = sync_globals_to_vars(pipeline)

# Sync only numeric types with prefix
report = sync_globals_to_vars(
    pipeline,
    include_types=[int, float],
    prefix="global_"
)

# Sync from specific namespace
import numpy as np
namespace = {"arr": np.array([1,2,3])}
report = sync_globals_to_vars(pipeline, namespace=namespace)

# Sync with deep copy and serialization check
report = sync_globals_to_vars(
    pipeline,
    deep_copy=True,
    only_serializable=True
)
    """
    # Initialize report
    report = SyncReport(
        added={},
        skipped={},
        errors={}
    )

    # Get namespace
    if namespace is None:
        # Get caller's globals
        namespace = currentframe().f_back.f_globals

    # Compile exclude patterns
    if exclude_patterns:
        patterns = [re.compile(pattern) for pattern in exclude_patterns]
    else:
        patterns = []

    # Normalize include_types
    if include_types and not isinstance(include_types, list | tuple | set):
        include_types = [include_types]
    def get_type_info(var: Any) -> str:
        """Helper to get detailed type information"""
        if isinstance(var, type):
            return f"class '{var.__name__}'"
        elif isinstance(var, BaseModel):
            return f"Pydantic model '{var.__class__.__name__}'"
        elif hasattr(var, '__class__'):
            type_name = var.__class__.__name__
            module_name = var.__class__.__module__
            if module_name != 'builtins':
                return f"{module_name}.{type_name}"
            return type_name
        return type(var).__name__
    # Process each variable
    for name, value in namespace.items():
        try:
            # Skip if matches exclude criteria
            if exclude_private and name.startswith('_'):
                report.skipped[name] = "private variable"
                continue

            if any(pattern.match(name) for pattern in patterns):
                report.skipped[name] = "matched exclude pattern"
                continue

            if include_types and not isinstance(value, tuple(include_types)):
                report.skipped[name] = f"type {type(value).__name__} not in include_types"
                continue

            # Test serialization if required
            if only_serializable:
                try:
                    import pickle
                    pickle.dumps(value)
                except Exception as e:
                    report.skipped[name] = f"not serializable: {str(e)}"
                    continue

            # Prepare variable
            var_value = deepcopy(value) if deep_copy else value
            var_name = f"{prefix}{name}" if prefix else name

            # Add to pipeline variables
            pipeline.variables[var_name] = var_value
            report.added[var_name] = get_type_info(value)

        except Exception as e:
            report.errors[name] = str(e)

    return report

base

Agent
agent
AgentCheckpoint dataclass

Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
@dataclass
class AgentCheckpoint:
    """Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration"""
    timestamp: datetime
    agent_state: dict[str, Any]
    task_state: dict[str, Any]
    world_model: dict[str, Any]
    active_flows: list[str]
    metadata: dict[str, Any] = field(default_factory=dict)

    # NEUE: Enhanced checkpoint data for UnifiedContextManager integration
    session_data: dict[str, Any] = field(default_factory=dict)
    context_manager_state: dict[str, Any] = field(default_factory=dict)
    conversation_history: list[dict[str, Any]] = field(default_factory=list)
    variable_system_state: dict[str, Any] = field(default_factory=dict)
    results_store: dict[str, Any] = field(default_factory=dict)
    tool_capabilities: dict[str, Any] = field(default_factory=dict)
    variable_scopes: dict[str, Any] = field(default_factory=dict)

    # Optional: Additional system state
    performance_metrics: dict[str, Any] = field(default_factory=dict)
    execution_history: list[dict[str, Any]] = field(default_factory=list)

    def get_checkpoint_summary(self) -> str:
        """Get human-readable checkpoint summary"""
        try:
            summary_parts = []

            # Basic info
            if self.session_data:
                session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
                summary_parts.append(f"{session_count} sessions")

            # Task info
            if self.task_state:
                completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
                total_tasks = len(self.task_state)
                summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

            # Conversation info
            if self.conversation_history:
                summary_parts.append(f"{len(self.conversation_history)} messages")

            # Context info
            if self.context_manager_state:
                cache_count = self.context_manager_state.get("cache_entries", 0)
                if cache_count > 0:
                    summary_parts.append(f"{cache_count} cached contexts")

            # Variable system info
            if self.variable_system_state:
                scopes = len(self.variable_system_state.get("scopes", {}))
                summary_parts.append(f"{scopes} variable scopes")

            # Tool capabilities
            if self.tool_capabilities:
                summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

            return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

        except Exception as e:
            return f"Summary generation failed: {str(e)}"

    def get_storage_size_estimate(self) -> dict[str, int]:
        """Estimate storage size of different checkpoint components"""
        try:
            sizes = {}

            # Calculate sizes in bytes (approximate)
            sizes["agent_state"] = len(str(self.agent_state))
            sizes["task_state"] = len(str(self.task_state))
            sizes["world_model"] = len(str(self.world_model))
            sizes["conversation_history"] = len(str(self.conversation_history))
            sizes["session_data"] = len(str(self.session_data))
            sizes["context_manager_state"] = len(str(self.context_manager_state))
            sizes["variable_system_state"] = len(str(self.variable_system_state))
            sizes["results_store"] = len(str(self.results_store))
            sizes["tool_capabilities"] = len(str(self.tool_capabilities))

            sizes["total_bytes"] = sum(sizes.values())
            sizes["total_kb"] = sizes["total_bytes"] / 1024
            sizes["total_mb"] = sizes["total_kb"] / 1024

            return sizes

        except Exception as e:
            return {"error": str(e)}

    def validate_checkpoint_integrity(self) -> dict[str, Any]:
        """Validate checkpoint integrity and completeness"""
        validation = {
            "is_valid": True,
            "errors": [],
            "warnings": [],
            "completeness_score": 0.0,
            "components_present": []
        }

        try:
            # Check required components
            required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
            for component in required_components:
                if hasattr(self, component) and getattr(self, component) is not None:
                    validation["components_present"].append(component)
                else:
                    validation["errors"].append(f"Missing required component: {component}")
                    validation["is_valid"] = False

            # Check optional enhanced components
            enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                                   "variable_system_state", "results_store", "tool_capabilities"]

            for component in enhanced_components:
                if hasattr(self, component) and getattr(self, component):
                    validation["components_present"].append(component)

            # Calculate completeness score
            total_possible = len(required_components) + len(enhanced_components)
            validation["completeness_score"] = len(validation["components_present"]) / total_possible

            # Check timestamp validity
            if isinstance(self.timestamp, datetime):
                age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
                if age_hours > 24:
                    validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
            else:
                validation["errors"].append("Invalid timestamp format")
                validation["is_valid"] = False

            # Check session data consistency
            if self.session_data and self.conversation_history:
                session_ids_in_data = set(self.session_data.keys())
                session_ids_in_conversation = set(
                    msg.get("session_id") for msg in self.conversation_history
                    if msg.get("session_id")
                )

                if session_ids_in_data != session_ids_in_conversation:
                    validation["warnings"].append("Session data and conversation history session IDs don't match")

            return validation

        except Exception as e:
            validation["errors"].append(f"Validation error: {str(e)}")
            validation["is_valid"] = False
            return validation

    def get_version_info(self) -> dict[str, str]:
        """Get checkpoint version information"""
        return {
            "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
            "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
            "context_system": "unified" if self.context_manager_state else "legacy",
            "variable_system": "integrated" if self.variable_system_state else "basic",
            "session_management": "chatsession" if self.session_data else "memory_only",
            "created_with": "FlowAgent v2.0 Enhanced Context System"
        }
get_checkpoint_summary()

Get human-readable checkpoint summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
def get_checkpoint_summary(self) -> str:
    """Get human-readable checkpoint summary"""
    try:
        summary_parts = []

        # Basic info
        if self.session_data:
            session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
            summary_parts.append(f"{session_count} sessions")

        # Task info
        if self.task_state:
            completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
            total_tasks = len(self.task_state)
            summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

        # Conversation info
        if self.conversation_history:
            summary_parts.append(f"{len(self.conversation_history)} messages")

        # Context info
        if self.context_manager_state:
            cache_count = self.context_manager_state.get("cache_entries", 0)
            if cache_count > 0:
                summary_parts.append(f"{cache_count} cached contexts")

        # Variable system info
        if self.variable_system_state:
            scopes = len(self.variable_system_state.get("scopes", {}))
            summary_parts.append(f"{scopes} variable scopes")

        # Tool capabilities
        if self.tool_capabilities:
            summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

        return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

    except Exception as e:
        return f"Summary generation failed: {str(e)}"
get_storage_size_estimate()

Estimate storage size of different checkpoint components

Source code in toolboxv2/mods/isaa/base/Agent/types.py
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
def get_storage_size_estimate(self) -> dict[str, int]:
    """Estimate storage size of different checkpoint components"""
    try:
        sizes = {}

        # Calculate sizes in bytes (approximate)
        sizes["agent_state"] = len(str(self.agent_state))
        sizes["task_state"] = len(str(self.task_state))
        sizes["world_model"] = len(str(self.world_model))
        sizes["conversation_history"] = len(str(self.conversation_history))
        sizes["session_data"] = len(str(self.session_data))
        sizes["context_manager_state"] = len(str(self.context_manager_state))
        sizes["variable_system_state"] = len(str(self.variable_system_state))
        sizes["results_store"] = len(str(self.results_store))
        sizes["tool_capabilities"] = len(str(self.tool_capabilities))

        sizes["total_bytes"] = sum(sizes.values())
        sizes["total_kb"] = sizes["total_bytes"] / 1024
        sizes["total_mb"] = sizes["total_kb"] / 1024

        return sizes

    except Exception as e:
        return {"error": str(e)}
get_version_info()

Get checkpoint version information

Source code in toolboxv2/mods/isaa/base/Agent/types.py
699
700
701
702
703
704
705
706
707
708
def get_version_info(self) -> dict[str, str]:
    """Get checkpoint version information"""
    return {
        "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
        "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
        "context_system": "unified" if self.context_manager_state else "legacy",
        "variable_system": "integrated" if self.variable_system_state else "basic",
        "session_management": "chatsession" if self.session_data else "memory_only",
        "created_with": "FlowAgent v2.0 Enhanced Context System"
    }
validate_checkpoint_integrity()

Validate checkpoint integrity and completeness

Source code in toolboxv2/mods/isaa/base/Agent/types.py
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
def validate_checkpoint_integrity(self) -> dict[str, Any]:
    """Validate checkpoint integrity and completeness"""
    validation = {
        "is_valid": True,
        "errors": [],
        "warnings": [],
        "completeness_score": 0.0,
        "components_present": []
    }

    try:
        # Check required components
        required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
        for component in required_components:
            if hasattr(self, component) and getattr(self, component) is not None:
                validation["components_present"].append(component)
            else:
                validation["errors"].append(f"Missing required component: {component}")
                validation["is_valid"] = False

        # Check optional enhanced components
        enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                               "variable_system_state", "results_store", "tool_capabilities"]

        for component in enhanced_components:
            if hasattr(self, component) and getattr(self, component):
                validation["components_present"].append(component)

        # Calculate completeness score
        total_possible = len(required_components) + len(enhanced_components)
        validation["completeness_score"] = len(validation["components_present"]) / total_possible

        # Check timestamp validity
        if isinstance(self.timestamp, datetime):
            age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
            if age_hours > 24:
                validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
        else:
            validation["errors"].append("Invalid timestamp format")
            validation["is_valid"] = False

        # Check session data consistency
        if self.session_data and self.conversation_history:
            session_ids_in_data = set(self.session_data.keys())
            session_ids_in_conversation = set(
                msg.get("session_id") for msg in self.conversation_history
                if msg.get("session_id")
            )

            if session_ids_in_data != session_ids_in_conversation:
                validation["warnings"].append("Session data and conversation history session IDs don't match")

        return validation

    except Exception as e:
        validation["errors"].append(f"Validation error: {str(e)}")
        validation["is_valid"] = False
        return validation
AgentModelData

Bases: BaseModel

Source code in toolboxv2/mods/isaa/base/Agent/types.py
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
class AgentModelData(BaseModel):
    name: str = "FlowAgent"
    fast_llm_model: str = "openrouter/anthropic/claude-3-haiku"
    complex_llm_model: str = "openrouter/openai/gpt-4o"
    system_message: str = "You are a production-ready autonomous agent."
    temperature: float = 0.7
    max_tokens: int = 2048
    max_input_tokens: int = 32768
    api_key: str | None  = None
    api_base: str | None  = None
    budget_manager: Any  = None
    caching: bool = True
    persona: PersonaConfig | None = True
    use_fast_response: bool = True

    def get_system_message_with_persona(self) -> str:
        """Get system message with persona integration"""
        base_message = self.system_message

        if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
            persona_addition = self.persona.to_system_prompt_addition()
            if persona_addition:
                base_message += f"\n## Persona Instructions\n{persona_addition}"

        return base_message
get_system_message_with_persona()

Get system message with persona integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
783
784
785
786
787
788
789
790
791
792
def get_system_message_with_persona(self) -> str:
    """Get system message with persona integration"""
    base_message = self.system_message

    if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
        persona_addition = self.persona.to_system_prompt_addition()
        if persona_addition:
            base_message += f"\n## Persona Instructions\n{persona_addition}"

    return base_message
BindingSyncHandler

Handles automatic synchronization between bound agents

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11783
11784
11785
11786
11787
11788
11789
11790
11791
11792
11793
11794
class BindingSyncHandler:
    """Handles automatic synchronization between bound agents"""

    def __init__(self, binding_config: dict):
        self.binding_config = binding_config
        self.sync_queue = []
        self.last_sync = time.time()

    def cleanup(self):
        """Clean up sync handler resources"""
        self.sync_queue.clear()
        rprint(f"Binding sync handler for {self.binding_config['binding_id']} cleaned up")
cleanup()

Clean up sync handler resources

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11791
11792
11793
11794
def cleanup(self):
    """Clean up sync handler resources"""
    self.sync_queue.clear()
    rprint(f"Binding sync handler for {self.binding_config['binding_id']} cleaned up")
ChainMetadata dataclass

Metadata for stored chains

Source code in toolboxv2/mods/isaa/base/Agent/types.py
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
@dataclass
class ChainMetadata:
    """Metadata for stored chains"""
    name: str
    description: str = ""
    created_at: datetime = field(default_factory=datetime.now)
    modified_at: datetime = field(default_factory=datetime.now)
    version: str = "1.0.0"
    tags: list[str] = field(default_factory=list)
    author: str = ""
    complexity: str = "simple"  # simple, medium, complex
    agent_count: int = 0
    has_conditionals: bool = False
    has_parallels: bool = False
    has_error_handling: bool = False
CompletionCheckerNode

Bases: AsyncNode

Breaks infinite cycles by checking actual completion status

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
@with_progress_tracking
class CompletionCheckerNode(AsyncNode):
    """Breaks infinite cycles by checking actual completion status"""

    def __init__(self):
        super().__init__()
        self.execution_count = 0
        self.max_cycles = 5  # Prevent infinite loops

    async def prep_async(self, shared):
        current_plan = shared.get("current_plan")
        tasks = shared.get("tasks", {})

        return {
            "current_plan": current_plan,
            "tasks": tasks,
            "execution_count": self.execution_count
        }

    async def exec_async(self, prep_res):
        self.execution_count += 1

        # Safety check: prevent infinite loops
        if self.execution_count > self.max_cycles:
            wprint(f"Max execution cycles ({self.max_cycles}) reached, terminating")
            return {
                "action": "force_terminate",
                "reason": "Max cycles reached"
            }

        current_plan = prep_res["current_plan"]
        tasks = prep_res["tasks"]

        if not current_plan:
            return {"action": "truly_complete", "reason": "No active plan"}

        # Check actual completion status
        pending_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "pending"]
        running_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "running"]
        completed_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "completed"]
        failed_tasks = [t for t in current_plan.tasks if tasks[t.id].status == "failed"]

        total_tasks = len(current_plan.tasks)

        # Truly complete: all tasks done
        if len(completed_tasks) + len(failed_tasks) == total_tasks:
            if len(failed_tasks) == 0 or len(completed_tasks) > len(failed_tasks):
                return {"action": "truly_complete", "reason": "All tasks completed"}
            else:
                return {"action": "truly_complete", "reason": "Plan failed but cannot continue"}

        # Has pending tasks that can run
        if pending_tasks and not running_tasks:
            return {"action": "continue_execution", "reason": f"{len(pending_tasks)} tasks ready"}

        # Has running tasks, wait
        if running_tasks:
            return {"action": "continue_execution", "reason": f"{len(running_tasks)} tasks running"}

        # Need reflection if tasks are stuck
        if pending_tasks and not running_tasks:
            return {"action": "needs_reflection", "reason": "Tasks may be blocked"}

        # Default: we're done
        return {"action": "truly_complete", "reason": "No actionable tasks"}

    async def post_async(self, shared, prep_res, exec_res):
        action = exec_res["action"]

        # Reset counter on true completion
        if action == "truly_complete":
            self.execution_count = 0
            shared["flow_completion_reason"] = exec_res["reason"]
        elif action == "force_terminate":  # HINZUGEFÜGT
            self.execution_count = 0
            shared["flow_completion_reason"] = f"Force terminated: {exec_res['reason']}"
            shared["force_terminated"] = True
            wprint(f"Flow force terminated: {exec_res['reason']}")

        return action
ContextAggregatorNode

Bases: AsyncNode

Vereinfachte Context-Aggregation über UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
@with_progress_tracking
class ContextAggregatorNode(AsyncNode):
    """Vereinfachte Context-Aggregation über UnifiedContextManager"""

    async def prep_async(self, shared):
        """Simplified preparation - delegate to UnifiedContextManager"""
        return {
            "context_manager": shared.get("context_manager"),
            "session_id": shared.get("session_id", "default"),
            "original_query": shared.get("current_query", ""),
            "tasks": shared.get("tasks", {}),
            "current_plan": shared.get("current_plan"),
            "world_model": shared.get("world_model", {}),
            "results": shared.get("results", {})
        }

    async def exec_async(self, prep_res):
        """VEREINFACHT: Get aggregated context from UnifiedContextManager"""

        context_manager = prep_res.get("context_manager")
        session_id = prep_res.get("session_id", "default")
        query = prep_res.get("original_query", "")

        if not context_manager:
            # Fallback: Create basic aggregated context
            return self._create_fallback_context(prep_res)

        try:
            #Get unified context from context manager
            unified_context = await context_manager.build_unified_context(session_id, query, "full")

            # Transform to expected aggregated_context format for compatibility
            aggregated_context = {
                "original_query": query,
                "successful_results": self._extract_successful_results(unified_context),
                "failed_attempts": self._extract_failed_attempts(prep_res["tasks"]),
                "key_discoveries": self._extract_key_discoveries(unified_context),
                "adaptation_summary": self._extract_adaptation_summary(prep_res),
                "confidence_scores": self._calculate_confidence_scores(unified_context),
                "unified_context": unified_context,  # Include full unified context
                "context_source": "unified_context_manager"
            }

            return aggregated_context

        except Exception as e:
            eprint(f"UnifiedContextManager aggregation failed: {e}")
            return self._create_fallback_context(prep_res)

    def _extract_successful_results(self, unified_context: dict[str, Any]) -> dict[str, Any]:
        """Extract successful results from unified context"""
        successful_results = {}

        try:
            # Get from variables context
            variables = unified_context.get("variables", {})
            recent_results = variables.get("recent_results", [])

            for result in recent_results:
                if result.get("success"):
                    task_id = result.get("task_id", f"result_{len(successful_results)}")
                    successful_results[task_id] = {
                        "task_description": f"Task {task_id}",
                        "task_type": "unified_context_result",
                        "result": result.get("preview", ""),
                        "metadata": {
                            "timestamp": result.get("timestamp"),
                            "source": "unified_context"
                        }
                    }

            # Also check execution state for completions
            execution_state = unified_context.get("execution_state", {})
            recent_completions = execution_state.get("recent_completions", [])

            for completion in recent_completions:
                task_id = completion.get("id", f"completion_{len(successful_results)}")
                successful_results[task_id] = {
                    "task_description": completion.get("description", "Completed task"),
                    "task_type": "execution_completion",
                    "result": f"Task completed at {completion.get('completed_at', 'unknown time')}",
                    "metadata": {
                        "completion_time": completion.get("completed_at"),
                        "source": "execution_state"
                    }
                }

            return successful_results

        except Exception as e:
            eprint(f"Error extracting successful results: {e}")
            return {}

    def _extract_failed_attempts(self, tasks: dict) -> dict[str, Any]:
        """Extract failed attempts from tasks (existing functionality)"""
        failed_attempts = {}

        try:
            for task_id, task in tasks.items():
                if task.status == "failed":
                    failed_attempts[task_id] = {
                        "description": task.description,
                        "error": task.error,
                        "retry_count": task.retry_count
                    }
            return failed_attempts
        except:
            return {}

    def _extract_key_discoveries(self, unified_context: dict[str, Any]) -> list[dict[str, Any]]:
        """Extract key discoveries from unified context"""
        discoveries = []

        try:
            # Extract from relevant facts
            relevant_facts = unified_context.get("relevant_facts", [])
            for key, value in relevant_facts[:3]:  # Top 3 facts
                discoveries.append({
                    "discovery": f"Fact discovered: {key}",
                    "confidence": 0.8,  # Default confidence for facts
                    "result": value
                })

            # Extract from successful results
            variables = unified_context.get("variables", {})
            recent_results = variables.get("recent_results", [])

            for result in recent_results[:2]:  # Top 2 results
                if result.get("success"):
                    discoveries.append({
                        "discovery": f"Task result: {result.get('task_id', 'unknown')}",
                        "confidence": 0.7,
                        "result": result.get("preview", "")
                    })

            return discoveries

        except Exception as e:
            eprint(f"Error extracting discoveries: {e}")
            return []

    def _extract_adaptation_summary(self, prep_res: dict) -> str:
        """Extract adaptation summary"""
        try:
            current_plan = prep_res.get("current_plan")
            if current_plan and hasattr(current_plan, 'metadata'):
                adaptations = current_plan.metadata.get("adaptations", 0)
                if adaptations > 0:
                    return f"Plan was adapted {adaptations} times to handle unexpected results."
            return ""
        except:
            return ""

    def _calculate_confidence_scores(self, unified_context: dict[str, Any]) -> dict[str, float]:
        """Calculate confidence scores based on unified context"""
        try:
            scores = {"overall": 0.5}

            # Base confidence on available data
            chat_history = unified_context.get("chat_history", [])
            if chat_history:
                scores["conversation_context"] = min(len(chat_history) / 10, 1.0)

            variables = unified_context.get("variables", {})
            recent_results = variables.get("recent_results", [])
            successful_results = [r for r in recent_results if r.get("success")]

            if recent_results:
                scores["execution_results"] = len(successful_results) / len(recent_results)

            # Calculate overall confidence
            scores["overall"] = sum(scores.values()) / len(scores)

            return scores

        except:
            return {"overall": 0.3}

    def _create_fallback_context(self, prep_res: dict) -> dict[str, Any]:
        """Create fallback context when UnifiedContextManager is unavailable"""
        return {
            "original_query": prep_res.get("original_query", ""),
            "successful_results": {},
            "failed_attempts": self._extract_failed_attempts(prep_res.get("tasks", {})),
            "key_discoveries": [],
            "adaptation_summary": "Fallback context - UnifiedContextManager unavailable",
            "confidence_scores": {"overall": 0.2},
            "context_source": "fallback"
        }

    async def post_async(self, shared, prep_res, exec_res):
        """Store aggregated context for downstream nodes"""
        shared["aggregated_context"] = exec_res

        #Also store unified context reference for other nodes
        if "unified_context" in exec_res:
            shared["unified_context"] = exec_res["unified_context"]

        if exec_res.get("successful_results") or exec_res.get("key_discoveries"):
            return "context_ready"
        else:
            return "no_context"
exec_async(prep_res) async

VEREINFACHT: Get aggregated context from UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
async def exec_async(self, prep_res):
    """VEREINFACHT: Get aggregated context from UnifiedContextManager"""

    context_manager = prep_res.get("context_manager")
    session_id = prep_res.get("session_id", "default")
    query = prep_res.get("original_query", "")

    if not context_manager:
        # Fallback: Create basic aggregated context
        return self._create_fallback_context(prep_res)

    try:
        #Get unified context from context manager
        unified_context = await context_manager.build_unified_context(session_id, query, "full")

        # Transform to expected aggregated_context format for compatibility
        aggregated_context = {
            "original_query": query,
            "successful_results": self._extract_successful_results(unified_context),
            "failed_attempts": self._extract_failed_attempts(prep_res["tasks"]),
            "key_discoveries": self._extract_key_discoveries(unified_context),
            "adaptation_summary": self._extract_adaptation_summary(prep_res),
            "confidence_scores": self._calculate_confidence_scores(unified_context),
            "unified_context": unified_context,  # Include full unified context
            "context_source": "unified_context_manager"
        }

        return aggregated_context

    except Exception as e:
        eprint(f"UnifiedContextManager aggregation failed: {e}")
        return self._create_fallback_context(prep_res)
post_async(shared, prep_res, exec_res) async

Store aggregated context for downstream nodes

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
async def post_async(self, shared, prep_res, exec_res):
    """Store aggregated context for downstream nodes"""
    shared["aggregated_context"] = exec_res

    #Also store unified context reference for other nodes
    if "unified_context" in exec_res:
        shared["unified_context"] = exec_res["unified_context"]

    if exec_res.get("successful_results") or exec_res.get("key_discoveries"):
        return "context_ready"
    else:
        return "no_context"
prep_async(shared) async

Simplified preparation - delegate to UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
async def prep_async(self, shared):
    """Simplified preparation - delegate to UnifiedContextManager"""
    return {
        "context_manager": shared.get("context_manager"),
        "session_id": shared.get("session_id", "default"),
        "original_query": shared.get("current_query", ""),
        "tasks": shared.get("tasks", {}),
        "current_plan": shared.get("current_plan"),
        "world_model": shared.get("world_model", {}),
        "results": shared.get("results", {})
    }
DecisionTask dataclass

Bases: Task

Task für dynamisches Routing

Source code in toolboxv2/mods/isaa/base/Agent/types.py
498
499
500
501
502
503
@dataclass
class DecisionTask(Task):
    """Task für dynamisches Routing"""
    decision_prompt: str = ""  # Kurze Frage an LLM
    routing_map: dict[str, str] = field(default_factory=dict)  # Ergebnis -> nächster Task
    decision_model: str = "fast"  # Welches LLM für Entscheidung
FlowAgent

Production-ready agent system built on PocketFlow

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
 8189
 8190
 8191
 8192
 8193
 8194
 8195
 8196
 8197
 8198
 8199
 8200
 8201
 8202
 8203
 8204
 8205
 8206
 8207
 8208
 8209
 8210
 8211
 8212
 8213
 8214
 8215
 8216
 8217
 8218
 8219
 8220
 8221
 8222
 8223
 8224
 8225
 8226
 8227
 8228
 8229
 8230
 8231
 8232
 8233
 8234
 8235
 8236
 8237
 8238
 8239
 8240
 8241
 8242
 8243
 8244
 8245
 8246
 8247
 8248
 8249
 8250
 8251
 8252
 8253
 8254
 8255
 8256
 8257
 8258
 8259
 8260
 8261
 8262
 8263
 8264
 8265
 8266
 8267
 8268
 8269
 8270
 8271
 8272
 8273
 8274
 8275
 8276
 8277
 8278
 8279
 8280
 8281
 8282
 8283
 8284
 8285
 8286
 8287
 8288
 8289
 8290
 8291
 8292
 8293
 8294
 8295
 8296
 8297
 8298
 8299
 8300
 8301
 8302
 8303
 8304
 8305
 8306
 8307
 8308
 8309
 8310
 8311
 8312
 8313
 8314
 8315
 8316
 8317
 8318
 8319
 8320
 8321
 8322
 8323
 8324
 8325
 8326
 8327
 8328
 8329
 8330
 8331
 8332
 8333
 8334
 8335
 8336
 8337
 8338
 8339
 8340
 8341
 8342
 8343
 8344
 8345
 8346
 8347
 8348
 8349
 8350
 8351
 8352
 8353
 8354
 8355
 8356
 8357
 8358
 8359
 8360
 8361
 8362
 8363
 8364
 8365
 8366
 8367
 8368
 8369
 8370
 8371
 8372
 8373
 8374
 8375
 8376
 8377
 8378
 8379
 8380
 8381
 8382
 8383
 8384
 8385
 8386
 8387
 8388
 8389
 8390
 8391
 8392
 8393
 8394
 8395
 8396
 8397
 8398
 8399
 8400
 8401
 8402
 8403
 8404
 8405
 8406
 8407
 8408
 8409
 8410
 8411
 8412
 8413
 8414
 8415
 8416
 8417
 8418
 8419
 8420
 8421
 8422
 8423
 8424
 8425
 8426
 8427
 8428
 8429
 8430
 8431
 8432
 8433
 8434
 8435
 8436
 8437
 8438
 8439
 8440
 8441
 8442
 8443
 8444
 8445
 8446
 8447
 8448
 8449
 8450
 8451
 8452
 8453
 8454
 8455
 8456
 8457
 8458
 8459
 8460
 8461
 8462
 8463
 8464
 8465
 8466
 8467
 8468
 8469
 8470
 8471
 8472
 8473
 8474
 8475
 8476
 8477
 8478
 8479
 8480
 8481
 8482
 8483
 8484
 8485
 8486
 8487
 8488
 8489
 8490
 8491
 8492
 8493
 8494
 8495
 8496
 8497
 8498
 8499
 8500
 8501
 8502
 8503
 8504
 8505
 8506
 8507
 8508
 8509
 8510
 8511
 8512
 8513
 8514
 8515
 8516
 8517
 8518
 8519
 8520
 8521
 8522
 8523
 8524
 8525
 8526
 8527
 8528
 8529
 8530
 8531
 8532
 8533
 8534
 8535
 8536
 8537
 8538
 8539
 8540
 8541
 8542
 8543
 8544
 8545
 8546
 8547
 8548
 8549
 8550
 8551
 8552
 8553
 8554
 8555
 8556
 8557
 8558
 8559
 8560
 8561
 8562
 8563
 8564
 8565
 8566
 8567
 8568
 8569
 8570
 8571
 8572
 8573
 8574
 8575
 8576
 8577
 8578
 8579
 8580
 8581
 8582
 8583
 8584
 8585
 8586
 8587
 8588
 8589
 8590
 8591
 8592
 8593
 8594
 8595
 8596
 8597
 8598
 8599
 8600
 8601
 8602
 8603
 8604
 8605
 8606
 8607
 8608
 8609
 8610
 8611
 8612
 8613
 8614
 8615
 8616
 8617
 8618
 8619
 8620
 8621
 8622
 8623
 8624
 8625
 8626
 8627
 8628
 8629
 8630
 8631
 8632
 8633
 8634
 8635
 8636
 8637
 8638
 8639
 8640
 8641
 8642
 8643
 8644
 8645
 8646
 8647
 8648
 8649
 8650
 8651
 8652
 8653
 8654
 8655
 8656
 8657
 8658
 8659
 8660
 8661
 8662
 8663
 8664
 8665
 8666
 8667
 8668
 8669
 8670
 8671
 8672
 8673
 8674
 8675
 8676
 8677
 8678
 8679
 8680
 8681
 8682
 8683
 8684
 8685
 8686
 8687
 8688
 8689
 8690
 8691
 8692
 8693
 8694
 8695
 8696
 8697
 8698
 8699
 8700
 8701
 8702
 8703
 8704
 8705
 8706
 8707
 8708
 8709
 8710
 8711
 8712
 8713
 8714
 8715
 8716
 8717
 8718
 8719
 8720
 8721
 8722
 8723
 8724
 8725
 8726
 8727
 8728
 8729
 8730
 8731
 8732
 8733
 8734
 8735
 8736
 8737
 8738
 8739
 8740
 8741
 8742
 8743
 8744
 8745
 8746
 8747
 8748
 8749
 8750
 8751
 8752
 8753
 8754
 8755
 8756
 8757
 8758
 8759
 8760
 8761
 8762
 8763
 8764
 8765
 8766
 8767
 8768
 8769
 8770
 8771
 8772
 8773
 8774
 8775
 8776
 8777
 8778
 8779
 8780
 8781
 8782
 8783
 8784
 8785
 8786
 8787
 8788
 8789
 8790
 8791
 8792
 8793
 8794
 8795
 8796
 8797
 8798
 8799
 8800
 8801
 8802
 8803
 8804
 8805
 8806
 8807
 8808
 8809
 8810
 8811
 8812
 8813
 8814
 8815
 8816
 8817
 8818
 8819
 8820
 8821
 8822
 8823
 8824
 8825
 8826
 8827
 8828
 8829
 8830
 8831
 8832
 8833
 8834
 8835
 8836
 8837
 8838
 8839
 8840
 8841
 8842
 8843
 8844
 8845
 8846
 8847
 8848
 8849
 8850
 8851
 8852
 8853
 8854
 8855
 8856
 8857
 8858
 8859
 8860
 8861
 8862
 8863
 8864
 8865
 8866
 8867
 8868
 8869
 8870
 8871
 8872
 8873
 8874
 8875
 8876
 8877
 8878
 8879
 8880
 8881
 8882
 8883
 8884
 8885
 8886
 8887
 8888
 8889
 8890
 8891
 8892
 8893
 8894
 8895
 8896
 8897
 8898
 8899
 8900
 8901
 8902
 8903
 8904
 8905
 8906
 8907
 8908
 8909
 8910
 8911
 8912
 8913
 8914
 8915
 8916
 8917
 8918
 8919
 8920
 8921
 8922
 8923
 8924
 8925
 8926
 8927
 8928
 8929
 8930
 8931
 8932
 8933
 8934
 8935
 8936
 8937
 8938
 8939
 8940
 8941
 8942
 8943
 8944
 8945
 8946
 8947
 8948
 8949
 8950
 8951
 8952
 8953
 8954
 8955
 8956
 8957
 8958
 8959
 8960
 8961
 8962
 8963
 8964
 8965
 8966
 8967
 8968
 8969
 8970
 8971
 8972
 8973
 8974
 8975
 8976
 8977
 8978
 8979
 8980
 8981
 8982
 8983
 8984
 8985
 8986
 8987
 8988
 8989
 8990
 8991
 8992
 8993
 8994
 8995
 8996
 8997
 8998
 8999
 9000
 9001
 9002
 9003
 9004
 9005
 9006
 9007
 9008
 9009
 9010
 9011
 9012
 9013
 9014
 9015
 9016
 9017
 9018
 9019
 9020
 9021
 9022
 9023
 9024
 9025
 9026
 9027
 9028
 9029
 9030
 9031
 9032
 9033
 9034
 9035
 9036
 9037
 9038
 9039
 9040
 9041
 9042
 9043
 9044
 9045
 9046
 9047
 9048
 9049
 9050
 9051
 9052
 9053
 9054
 9055
 9056
 9057
 9058
 9059
 9060
 9061
 9062
 9063
 9064
 9065
 9066
 9067
 9068
 9069
 9070
 9071
 9072
 9073
 9074
 9075
 9076
 9077
 9078
 9079
 9080
 9081
 9082
 9083
 9084
 9085
 9086
 9087
 9088
 9089
 9090
 9091
 9092
 9093
 9094
 9095
 9096
 9097
 9098
 9099
 9100
 9101
 9102
 9103
 9104
 9105
 9106
 9107
 9108
 9109
 9110
 9111
 9112
 9113
 9114
 9115
 9116
 9117
 9118
 9119
 9120
 9121
 9122
 9123
 9124
 9125
 9126
 9127
 9128
 9129
 9130
 9131
 9132
 9133
 9134
 9135
 9136
 9137
 9138
 9139
 9140
 9141
 9142
 9143
 9144
 9145
 9146
 9147
 9148
 9149
 9150
 9151
 9152
 9153
 9154
 9155
 9156
 9157
 9158
 9159
 9160
 9161
 9162
 9163
 9164
 9165
 9166
 9167
 9168
 9169
 9170
 9171
 9172
 9173
 9174
 9175
 9176
 9177
 9178
 9179
 9180
 9181
 9182
 9183
 9184
 9185
 9186
 9187
 9188
 9189
 9190
 9191
 9192
 9193
 9194
 9195
 9196
 9197
 9198
 9199
 9200
 9201
 9202
 9203
 9204
 9205
 9206
 9207
 9208
 9209
 9210
 9211
 9212
 9213
 9214
 9215
 9216
 9217
 9218
 9219
 9220
 9221
 9222
 9223
 9224
 9225
 9226
 9227
 9228
 9229
 9230
 9231
 9232
 9233
 9234
 9235
 9236
 9237
 9238
 9239
 9240
 9241
 9242
 9243
 9244
 9245
 9246
 9247
 9248
 9249
 9250
 9251
 9252
 9253
 9254
 9255
 9256
 9257
 9258
 9259
 9260
 9261
 9262
 9263
 9264
 9265
 9266
 9267
 9268
 9269
 9270
 9271
 9272
 9273
 9274
 9275
 9276
 9277
 9278
 9279
 9280
 9281
 9282
 9283
 9284
 9285
 9286
 9287
 9288
 9289
 9290
 9291
 9292
 9293
 9294
 9295
 9296
 9297
 9298
 9299
 9300
 9301
 9302
 9303
 9304
 9305
 9306
 9307
 9308
 9309
 9310
 9311
 9312
 9313
 9314
 9315
 9316
 9317
 9318
 9319
 9320
 9321
 9322
 9323
 9324
 9325
 9326
 9327
 9328
 9329
 9330
 9331
 9332
 9333
 9334
 9335
 9336
 9337
 9338
 9339
 9340
 9341
 9342
 9343
 9344
 9345
 9346
 9347
 9348
 9349
 9350
 9351
 9352
 9353
 9354
 9355
 9356
 9357
 9358
 9359
 9360
 9361
 9362
 9363
 9364
 9365
 9366
 9367
 9368
 9369
 9370
 9371
 9372
 9373
 9374
 9375
 9376
 9377
 9378
 9379
 9380
 9381
 9382
 9383
 9384
 9385
 9386
 9387
 9388
 9389
 9390
 9391
 9392
 9393
 9394
 9395
 9396
 9397
 9398
 9399
 9400
 9401
 9402
 9403
 9404
 9405
 9406
 9407
 9408
 9409
 9410
 9411
 9412
 9413
 9414
 9415
 9416
 9417
 9418
 9419
 9420
 9421
 9422
 9423
 9424
 9425
 9426
 9427
 9428
 9429
 9430
 9431
 9432
 9433
 9434
 9435
 9436
 9437
 9438
 9439
 9440
 9441
 9442
 9443
 9444
 9445
 9446
 9447
 9448
 9449
 9450
 9451
 9452
 9453
 9454
 9455
 9456
 9457
 9458
 9459
 9460
 9461
 9462
 9463
 9464
 9465
 9466
 9467
 9468
 9469
 9470
 9471
 9472
 9473
 9474
 9475
 9476
 9477
 9478
 9479
 9480
 9481
 9482
 9483
 9484
 9485
 9486
 9487
 9488
 9489
 9490
 9491
 9492
 9493
 9494
 9495
 9496
 9497
 9498
 9499
 9500
 9501
 9502
 9503
 9504
 9505
 9506
 9507
 9508
 9509
 9510
 9511
 9512
 9513
 9514
 9515
 9516
 9517
 9518
 9519
 9520
 9521
 9522
 9523
 9524
 9525
 9526
 9527
 9528
 9529
 9530
 9531
 9532
 9533
 9534
 9535
 9536
 9537
 9538
 9539
 9540
 9541
 9542
 9543
 9544
 9545
 9546
 9547
 9548
 9549
 9550
 9551
 9552
 9553
 9554
 9555
 9556
 9557
 9558
 9559
 9560
 9561
 9562
 9563
 9564
 9565
 9566
 9567
 9568
 9569
 9570
 9571
 9572
 9573
 9574
 9575
 9576
 9577
 9578
 9579
 9580
 9581
 9582
 9583
 9584
 9585
 9586
 9587
 9588
 9589
 9590
 9591
 9592
 9593
 9594
 9595
 9596
 9597
 9598
 9599
 9600
 9601
 9602
 9603
 9604
 9605
 9606
 9607
 9608
 9609
 9610
 9611
 9612
 9613
 9614
 9615
 9616
 9617
 9618
 9619
 9620
 9621
 9622
 9623
 9624
 9625
 9626
 9627
 9628
 9629
 9630
 9631
 9632
 9633
 9634
 9635
 9636
 9637
 9638
 9639
 9640
 9641
 9642
 9643
 9644
 9645
 9646
 9647
 9648
 9649
 9650
 9651
 9652
 9653
 9654
 9655
 9656
 9657
 9658
 9659
 9660
 9661
 9662
 9663
 9664
 9665
 9666
 9667
 9668
 9669
 9670
 9671
 9672
 9673
 9674
 9675
 9676
 9677
 9678
 9679
 9680
 9681
 9682
 9683
 9684
 9685
 9686
 9687
 9688
 9689
 9690
 9691
 9692
 9693
 9694
 9695
 9696
 9697
 9698
 9699
 9700
 9701
 9702
 9703
 9704
 9705
 9706
 9707
 9708
 9709
 9710
 9711
 9712
 9713
 9714
 9715
 9716
 9717
 9718
 9719
 9720
 9721
 9722
 9723
 9724
 9725
 9726
 9727
 9728
 9729
 9730
 9731
 9732
 9733
 9734
 9735
 9736
 9737
 9738
 9739
 9740
 9741
 9742
 9743
 9744
 9745
 9746
 9747
 9748
 9749
 9750
 9751
 9752
 9753
 9754
 9755
 9756
 9757
 9758
 9759
 9760
 9761
 9762
 9763
 9764
 9765
 9766
 9767
 9768
 9769
 9770
 9771
 9772
 9773
 9774
 9775
 9776
 9777
 9778
 9779
 9780
 9781
 9782
 9783
 9784
 9785
 9786
 9787
 9788
 9789
 9790
 9791
 9792
 9793
 9794
 9795
 9796
 9797
 9798
 9799
 9800
 9801
 9802
 9803
 9804
 9805
 9806
 9807
 9808
 9809
 9810
 9811
 9812
 9813
 9814
 9815
 9816
 9817
 9818
 9819
 9820
 9821
 9822
 9823
 9824
 9825
 9826
 9827
 9828
 9829
 9830
 9831
 9832
 9833
 9834
 9835
 9836
 9837
 9838
 9839
 9840
 9841
 9842
 9843
 9844
 9845
 9846
 9847
 9848
 9849
 9850
 9851
 9852
 9853
 9854
 9855
 9856
 9857
 9858
 9859
 9860
 9861
 9862
 9863
 9864
 9865
 9866
 9867
 9868
 9869
 9870
 9871
 9872
 9873
 9874
 9875
 9876
 9877
 9878
 9879
 9880
 9881
 9882
 9883
 9884
 9885
 9886
 9887
 9888
 9889
 9890
 9891
 9892
 9893
 9894
 9895
 9896
 9897
 9898
 9899
 9900
 9901
 9902
 9903
 9904
 9905
 9906
 9907
 9908
 9909
 9910
 9911
 9912
 9913
 9914
 9915
 9916
 9917
 9918
 9919
 9920
 9921
 9922
 9923
 9924
 9925
 9926
 9927
 9928
 9929
 9930
 9931
 9932
 9933
 9934
 9935
 9936
 9937
 9938
 9939
 9940
 9941
 9942
 9943
 9944
 9945
 9946
 9947
 9948
 9949
 9950
 9951
 9952
 9953
 9954
 9955
 9956
 9957
 9958
 9959
 9960
 9961
 9962
 9963
 9964
 9965
 9966
 9967
 9968
 9969
 9970
 9971
 9972
 9973
 9974
 9975
 9976
 9977
 9978
 9979
 9980
 9981
 9982
 9983
 9984
 9985
 9986
 9987
 9988
 9989
 9990
 9991
 9992
 9993
 9994
 9995
 9996
 9997
 9998
 9999
10000
10001
10002
10003
10004
10005
10006
10007
10008
10009
10010
10011
10012
10013
10014
10015
10016
10017
10018
10019
10020
10021
10022
10023
10024
10025
10026
10027
10028
10029
10030
10031
10032
10033
10034
10035
10036
10037
10038
10039
10040
10041
10042
10043
10044
10045
10046
10047
10048
10049
10050
10051
10052
10053
10054
10055
10056
10057
10058
10059
10060
10061
10062
10063
10064
10065
10066
10067
10068
10069
10070
10071
10072
10073
10074
10075
10076
10077
10078
10079
10080
10081
10082
10083
10084
10085
10086
10087
10088
10089
10090
10091
10092
10093
10094
10095
10096
10097
10098
10099
10100
10101
10102
10103
10104
10105
10106
10107
10108
10109
10110
10111
10112
10113
10114
10115
10116
10117
10118
10119
10120
10121
10122
10123
10124
10125
10126
10127
10128
10129
10130
10131
10132
10133
10134
10135
10136
10137
10138
10139
10140
10141
10142
10143
10144
10145
10146
10147
10148
10149
10150
10151
10152
10153
10154
10155
10156
10157
10158
10159
10160
10161
10162
10163
10164
10165
10166
10167
10168
10169
10170
10171
10172
10173
10174
10175
10176
10177
10178
10179
10180
10181
10182
10183
10184
10185
10186
10187
10188
10189
10190
10191
10192
10193
10194
10195
10196
10197
10198
10199
10200
10201
10202
10203
10204
10205
10206
10207
10208
10209
10210
10211
10212
10213
10214
10215
10216
10217
10218
10219
10220
10221
10222
10223
10224
10225
10226
10227
10228
10229
10230
10231
10232
10233
10234
10235
10236
10237
10238
10239
10240
10241
10242
10243
10244
10245
10246
10247
10248
10249
10250
10251
10252
10253
10254
10255
10256
10257
10258
10259
10260
10261
10262
10263
10264
10265
10266
10267
10268
10269
10270
10271
10272
10273
10274
10275
10276
10277
10278
10279
10280
10281
10282
10283
10284
10285
10286
10287
10288
10289
10290
10291
10292
10293
10294
10295
10296
10297
10298
10299
10300
10301
10302
10303
10304
10305
10306
10307
10308
10309
10310
10311
10312
10313
10314
10315
10316
10317
10318
10319
10320
10321
10322
10323
10324
10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
10340
10341
10342
10343
10344
10345
10346
10347
10348
10349
10350
10351
10352
10353
10354
10355
10356
10357
10358
10359
10360
10361
10362
10363
10364
10365
10366
10367
10368
10369
10370
10371
10372
10373
10374
10375
10376
10377
10378
10379
10380
10381
10382
10383
10384
10385
10386
10387
10388
10389
10390
10391
10392
10393
10394
10395
10396
10397
10398
10399
10400
10401
10402
10403
10404
10405
10406
10407
10408
10409
10410
10411
10412
10413
10414
10415
10416
10417
10418
10419
10420
10421
10422
10423
10424
10425
10426
10427
10428
10429
10430
10431
10432
10433
10434
10435
10436
10437
10438
10439
10440
10441
10442
10443
10444
10445
10446
10447
10448
10449
10450
10451
10452
10453
10454
10455
10456
10457
10458
10459
10460
10461
10462
10463
10464
10465
10466
10467
10468
10469
10470
10471
10472
10473
10474
10475
10476
10477
10478
10479
10480
10481
10482
10483
10484
10485
10486
10487
10488
10489
10490
10491
10492
10493
10494
10495
10496
10497
10498
10499
10500
10501
10502
10503
10504
10505
10506
10507
10508
10509
10510
10511
10512
10513
10514
10515
10516
10517
10518
10519
10520
10521
10522
10523
10524
10525
10526
10527
10528
10529
10530
10531
10532
10533
10534
10535
10536
10537
10538
10539
10540
10541
10542
10543
10544
10545
10546
10547
10548
10549
10550
10551
10552
10553
10554
10555
10556
10557
10558
10559
10560
10561
10562
10563
10564
10565
10566
10567
10568
10569
10570
10571
10572
10573
10574
10575
10576
10577
10578
10579
10580
10581
10582
10583
10584
10585
10586
10587
10588
10589
10590
10591
10592
10593
10594
10595
10596
10597
10598
10599
10600
10601
10602
10603
10604
10605
10606
10607
10608
10609
10610
10611
10612
10613
10614
10615
10616
10617
10618
10619
10620
10621
10622
10623
10624
10625
10626
10627
10628
10629
10630
10631
10632
10633
10634
10635
10636
10637
10638
10639
10640
10641
10642
10643
10644
10645
10646
10647
10648
10649
10650
10651
10652
10653
10654
10655
10656
10657
10658
10659
10660
10661
10662
10663
10664
10665
10666
10667
10668
10669
10670
10671
10672
10673
10674
10675
10676
10677
10678
10679
10680
10681
10682
10683
10684
10685
10686
10687
10688
10689
10690
10691
10692
10693
10694
10695
10696
10697
10698
10699
10700
10701
10702
10703
10704
10705
10706
10707
10708
10709
10710
10711
10712
10713
10714
10715
10716
10717
10718
10719
10720
10721
10722
10723
10724
10725
10726
10727
10728
10729
10730
10731
10732
10733
10734
10735
10736
10737
10738
10739
10740
10741
10742
10743
10744
10745
10746
10747
10748
10749
10750
10751
10752
10753
10754
10755
10756
10757
10758
10759
10760
10761
10762
10763
10764
10765
10766
10767
10768
10769
10770
10771
10772
10773
10774
10775
10776
10777
10778
10779
10780
10781
10782
10783
10784
10785
10786
10787
10788
10789
10790
10791
10792
10793
10794
10795
10796
10797
10798
10799
10800
10801
10802
10803
10804
10805
10806
10807
10808
10809
10810
10811
10812
10813
10814
10815
10816
10817
10818
10819
10820
10821
10822
10823
10824
10825
10826
10827
10828
10829
10830
10831
10832
10833
10834
10835
10836
10837
10838
10839
10840
10841
10842
10843
10844
10845
10846
10847
10848
10849
10850
10851
10852
10853
10854
10855
10856
10857
10858
10859
10860
10861
10862
10863
10864
10865
10866
10867
10868
10869
10870
10871
10872
10873
10874
10875
10876
10877
10878
10879
10880
10881
10882
10883
10884
10885
10886
10887
10888
10889
10890
10891
10892
10893
10894
10895
10896
10897
10898
10899
10900
10901
10902
10903
10904
10905
10906
10907
10908
10909
10910
10911
10912
10913
10914
10915
10916
10917
10918
10919
10920
10921
10922
10923
10924
10925
10926
10927
10928
10929
10930
10931
10932
10933
10934
10935
10936
10937
10938
10939
10940
10941
10942
10943
10944
10945
10946
10947
10948
10949
10950
10951
10952
10953
10954
10955
10956
10957
10958
10959
10960
10961
10962
10963
10964
10965
10966
10967
10968
10969
10970
10971
10972
10973
10974
10975
10976
10977
10978
10979
10980
10981
10982
10983
10984
10985
10986
10987
10988
10989
10990
10991
10992
10993
10994
10995
10996
10997
10998
10999
11000
11001
11002
11003
11004
11005
11006
11007
11008
11009
11010
11011
11012
11013
11014
11015
11016
11017
11018
11019
11020
11021
11022
11023
11024
11025
11026
11027
11028
11029
11030
11031
11032
11033
11034
11035
11036
11037
11038
11039
11040
11041
11042
11043
11044
11045
11046
11047
11048
11049
11050
11051
11052
11053
11054
11055
11056
11057
11058
11059
11060
11061
11062
11063
11064
11065
11066
11067
11068
11069
11070
11071
11072
11073
11074
11075
11076
11077
11078
11079
11080
11081
11082
11083
11084
11085
11086
11087
11088
11089
11090
11091
11092
11093
11094
11095
11096
11097
11098
11099
11100
11101
11102
11103
11104
11105
11106
11107
11108
11109
11110
11111
11112
11113
11114
11115
11116
11117
11118
11119
11120
11121
11122
11123
11124
11125
11126
11127
11128
11129
11130
11131
11132
11133
11134
11135
11136
11137
11138
11139
11140
11141
11142
11143
11144
11145
11146
11147
11148
11149
11150
11151
11152
11153
11154
11155
11156
11157
11158
11159
11160
11161
11162
11163
11164
11165
11166
11167
11168
11169
11170
11171
11172
11173
11174
11175
11176
11177
11178
11179
11180
11181
11182
11183
11184
11185
11186
11187
11188
11189
11190
11191
11192
11193
11194
11195
11196
11197
11198
11199
11200
11201
11202
11203
11204
11205
11206
11207
11208
11209
11210
11211
11212
11213
11214
11215
11216
11217
11218
11219
11220
11221
11222
11223
11224
11225
11226
11227
11228
11229
11230
11231
11232
11233
11234
11235
11236
11237
11238
11239
11240
11241
11242
11243
11244
11245
11246
11247
11248
11249
11250
11251
11252
11253
11254
11255
11256
11257
11258
11259
11260
11261
11262
11263
11264
11265
11266
11267
11268
11269
11270
11271
11272
11273
11274
11275
11276
11277
11278
11279
11280
11281
11282
11283
11284
11285
11286
11287
11288
11289
11290
11291
11292
11293
11294
11295
11296
11297
11298
11299
11300
11301
11302
11303
11304
11305
11306
11307
11308
11309
11310
11311
11312
11313
11314
11315
11316
11317
11318
11319
11320
11321
11322
11323
11324
11325
11326
11327
11328
11329
11330
11331
11332
11333
11334
11335
11336
11337
11338
11339
11340
11341
11342
11343
11344
11345
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356
11357
11358
11359
11360
11361
11362
11363
11364
11365
11366
11367
11368
11369
11370
11371
11372
11373
11374
11375
11376
11377
11378
11379
11380
11381
11382
11383
11384
11385
11386
11387
11388
11389
11390
11391
11392
11393
11394
11395
11396
11397
11398
11399
11400
11401
11402
11403
11404
11405
11406
11407
11408
11409
11410
11411
11412
11413
11414
11415
11416
11417
11418
11419
11420
11421
11422
11423
11424
11425
11426
11427
11428
11429
11430
11431
11432
11433
11434
11435
11436
11437
11438
11439
11440
11441
11442
11443
11444
11445
11446
11447
11448
11449
11450
11451
11452
11453
11454
11455
11456
11457
11458
11459
11460
11461
11462
11463
11464
11465
11466
11467
11468
11469
11470
11471
11472
11473
11474
11475
11476
11477
11478
11479
11480
11481
11482
11483
11484
11485
11486
11487
11488
11489
11490
11491
11492
11493
11494
11495
11496
11497
11498
11499
11500
11501
11502
11503
11504
11505
11506
11507
11508
11509
11510
11511
11512
11513
11514
11515
11516
11517
11518
11519
11520
11521
11522
11523
11524
11525
11526
11527
11528
11529
11530
11531
11532
11533
11534
11535
11536
11537
11538
11539
11540
11541
11542
11543
11544
11545
11546
11547
11548
11549
11550
11551
11552
11553
11554
11555
11556
11557
11558
11559
11560
11561
11562
11563
11564
11565
11566
11567
11568
11569
11570
11571
11572
11573
11574
11575
11576
11577
11578
11579
11580
11581
11582
11583
11584
11585
11586
11587
11588
11589
11590
11591
11592
11593
11594
11595
11596
11597
11598
11599
11600
11601
11602
11603
11604
11605
11606
11607
11608
11609
11610
11611
11612
11613
11614
11615
11616
11617
11618
11619
11620
11621
11622
11623
11624
11625
11626
11627
11628
11629
11630
11631
11632
11633
11634
11635
11636
11637
11638
11639
11640
11641
11642
11643
11644
11645
11646
11647
11648
11649
11650
11651
11652
11653
11654
11655
11656
11657
11658
11659
11660
11661
11662
11663
11664
11665
11666
11667
11668
11669
11670
11671
11672
11673
11674
11675
11676
11677
11678
11679
11680
11681
11682
11683
11684
11685
11686
11687
11688
11689
11690
class FlowAgent:
    """Production-ready agent system built on PocketFlow """
    def __init__(
        self,
        amd: AgentModelData,
        world_model: dict[str, Any] = None,
        verbose: bool = False,
        enable_pause_resume: bool = True,
        checkpoint_interval: int = 300,  # 5 minutes
        max_parallel_tasks: int = 3,
        progress_callback: callable = None,
        stream:bool=True,
        **kwargs
    ):
        self.amd = amd
        self.stream = stream
        self.world_model = world_model or {}
        self.verbose = verbose
        self.enable_pause_resume = enable_pause_resume
        self.checkpoint_interval = checkpoint_interval
        self.max_parallel_tasks = max_parallel_tasks
        self.progress_tracker = ProgressTracker(progress_callback, agent_name=amd.name)

        # Core state
        self.shared = {
            "world_model": self.world_model,
            "tasks": {},
            "task_plans": {},
            "system_status": "idle",
            "session_data": {},
            "performance_metrics": {},
            "conversation_history": [],
            "available_tools": [],
            "progress_tracker": self.progress_tracker
        }
        self.context_manager = UnifiedContextManager(self)
        self.variable_manager = VariableManager(self.shared["world_model"], self.shared)
        self.context_manager.variable_manager = self.variable_manager# Register default scopes

        self.shared["context_manager"] = self.context_manager
        self.shared["variable_manager"] = self.variable_manager
        # Flows
        self.task_flow = TaskManagementFlow(max_parallel_tasks=self.max_parallel_tasks)
        self.response_flow = ResponseGenerationFlow()

        if hasattr(self.task_flow, 'executor_node'):
            self.task_flow.executor_node.agent_instance = self

        # Agent state
        self.is_running = False
        self.is_paused = False
        self.last_checkpoint = None
        self.checkpoint_data = {}
        self.ac_cost = 0

        # Threading
        self.executor = ThreadPoolExecutor(max_workers=max_parallel_tasks)
        self._shutdown_event = threading.Event()

        # Server components
        self.a2a_server: A2AServer = None
        self.mcp_server: FastMCP = None

        # Enhanced tool registry
        self._tool_registry = {}
        self._tool_capabilities = {}
        self._tool_analysis_cache = {}

        self.active_session = None
        # Tool analysis file path
        self.tool_analysis_file = self._get_tool_analysis_path()

        self._tool_capabilities.update(self._load_tool_analysis())
        if self.amd.budget_manager:
            self.amd.budget_manager.load_data()

        self._setup_variable_scopes()

        rprint(f"FlowAgent initialized: {amd.name}")

    def task_flow_settings(self, max_parallel_tasks: int = 3, max_reasoning_loops: int = 24, max_tool_calls:int = 5):
        self.task_flow.executor_node.max_parallel = max_parallel_tasks
        self.task_flow.llm_reasoner.max_reasoning_loops = max_reasoning_loops
        self.task_flow.llm_tool_node.max_tool_calls = max_tool_calls

    @property
    def progress_callback(self):
        return self.progress_tracker.progress_callback

    @progress_callback.setter
    def progress_callback(self, value):
        self.progress_tracker.progress_callback = value

    def set_progress_callback(self, progress_callback: callable = None):
        self.progress_callback = progress_callback

    async def a_run_llm_completion(self, node_name="FlowAgentLLMCall",task_id="unknown",model_preference="fast", with_context=True, auto_fallbacks=True, **kwargs) -> str:
        if "model" not in kwargs:
            kwargs["model"] = self.amd.fast_llm_model if model_preference == "fast" else self.amd.complex_llm_model

        if not 'stream' in kwargs:
            kwargs['stream'] = self.stream

        llm_start = time.perf_counter()

        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="llm_call",
                node_name=node_name,
                session_id=self.active_session,
                task_id=task_id,
                status=NodeStatus.RUNNING,
                llm_model=kwargs["model"],
                llm_temperature=kwargs.get("temperature", 0.7),
                llm_input=kwargs.get("messages", [{}])[-1].get("content", ""),  # Prompt direkt erfassen
                metadata={
                    "model_preference": kwargs.get("model_preference", "fast")
                }
            ))

        # auto api key addition supports (google, openrouter, openai, anthropic, azure, aws, huggingface, replicate, togetherai, groq)
        if "api_key" not in kwargs:
            # litellm model-prefix apikey mapp
            prefix = kwargs['model'].split("/")[0]
            model_prefix_map = {
                "openrouter": os.getenv("OPENROUTER_API_KEY"),
                "openai": os.getenv("OPENAI_API_KEY"),
                "anthropic": os.getenv("ANTHROPIC_API_KEY"),
                "google": os.getenv("GOOGLE_API_KEY"),
                "azure": os.getenv("AZURE_API_KEY"),
                "huggingface": os.getenv("HUGGINGFACE_API_KEY"),
                "replicate": os.getenv("REPLICATE_API_KEY"),
                "togetherai": os.getenv("TOGETHERAI_API_KEY"),
                "groq": os.getenv("GROQ_API_KEY"),
            }
            kwargs["api_key"] = model_prefix_map.get(prefix)

        if self.active_session and with_context:
            # Add context to fist messages as system message
            context_ = await self.get_context(self.active_session)
            kwargs["messages"] = [{"role": "system", "content": self.amd.get_system_message_with_persona()+'\n\nContext:\n\n'+context_}] + kwargs.get("messages", [])

        # build fallback dict using FALLBACKS_MODELS/PREM and _KEYS

        if auto_fallbacks and 'fallbacks' not in kwargs:
            fallbacks_dict_list = []
            fallbacks = os.getenv("FALLBACKS_MODELS", '').split(',') if model_preference == "fast" else os.getenv(
                "FALLBACKS_MODELS_PREM", '').split(',')
            fallbacks_keys = os.getenv("FALLBACKS_MODELS_KEYS", '').split(
                ',') if model_preference == "fast" else os.getenv(
                "FALLBACKS_MODELS_KEYS_PREM", '').split(',')
            for model, key in zip(fallbacks, fallbacks_keys):
                fallbacks_dict_list.append({"model": model, "api_key": os.getenv(key, kwargs.get("api_key", None))})
            kwargs['fallbacks'] = fallbacks_dict_list
        try:

            if kwargs.get("stream", False):
                kwargs["stream_options"] = {"include_usage": True}

            # detailed informations str
            with Spinner(f"LLM Call {self.amd.name}@{node_name}#{task_id if task_id else model_preference}-{kwargs['model']}"):
                response = await litellm.acompletion(**kwargs)

            if not kwargs.get("stream", False):
                result = response.choices[0].message.content
                usage = response.usage
                input_tokens = usage.prompt_tokens if usage else 0
                output_tokens = usage.completion_tokens if usage else 0
                total_tokens = usage.total_tokens if usage else 0

            else:
                result = ""
                final_chunk = None
                async for chunk in response:
                    content = chunk.choices[0].delta.content or ""
                    result += content
                    if self.progress_tracker and content:
                        await self.progress_tracker.emit_event(ProgressEvent(
                            event_type="llm_stream_chunk",
                            node_name=node_name,
                            task_id=task_id,
                            session_id=self.active_session,
                            status=NodeStatus.RUNNING,
                            # weitere Felder wie model, tokens wenn verfügbar
                            llm_model=kwargs["model"],
                            llm_output=content,
                            # optional: llm_tokens so far? usage in chunk?
                        ))
                    final_chunk = chunk

                usage = final_chunk.usage if hasattr(final_chunk, "usage") else None
                output_tokens = usage.completion_tokens if usage else 0
                input_tokens = usage.prompt_tokens if usage else 0
                total_tokens = usage.total_tokens if usage else 0
            llm_duration = time.perf_counter() - llm_start

            if AGENT_VERBOSE and self.verbose:
                kwargs["messages"] += [{"role": "assistant", "content": result}]
                print_prompt(kwargs)
            # else:
            #     print_prompt([{"role": "assistant", "content": result}])

            # Extract token usage and cost


            call_cost = self.progress_tracker.calculate_llm_cost(kwargs["model"], input_tokens,
                                                            output_tokens, response) if self.progress_tracker else 0.0
            self.ac_cost += call_cost
            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",
                    node_name=node_name,
                    task_id=task_id,
                    session_id=self.active_session,
                    status=NodeStatus.COMPLETED,
                    success=True,
                    duration=llm_duration,
                    llm_model=kwargs["model"],
                    llm_prompt_tokens=input_tokens,
                    llm_completion_tokens=output_tokens,
                    llm_total_tokens=total_tokens,
                    llm_cost=call_cost,
                    llm_temperature=kwargs.get("temperature", 0.7),
                    llm_output=result,
                    llm_input="",
                ))

            return result
        except Exception as e:
            llm_duration = time.perf_counter() - llm_start
            import traceback
            print(traceback.format_exc())
            # print(f"LLM call failed: {json.dumps(kwargs, indent=2)}")

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",  # Event-Typ bleibt konsistent
                    node_name=node_name,
                    task_id=task_id,
                    session_id=self.active_session,
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=llm_duration,
                    llm_model=kwargs["model"],
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    }
                ))

            raise

    async def a_run(
        self,
        query: str,
        session_id: str = "default",
        user_id: str = None,
        stream_callback: Callable = None,
        remember: bool = True,
        **kwargs
    ) -> str:
        """Main entry point für Agent-Ausführung mit UnifiedContextManager"""

        execution_start = self.progress_tracker.start_timer("total_execution")
        self.active_session = session_id
        result = None
        await self.progress_tracker.emit_event(ProgressEvent(
            event_type="execution_start",
            timestamp=time.time(),
            status=NodeStatus.RUNNING,
            node_name="FlowAgent",
            session_id=session_id,
            metadata={"query": query, "user_id": user_id}
        ))

        try:
            #Initialize or get session über UnifiedContextManager
            await self.initialize_session_context(session_id, max_history=200)

            #Store user message immediately in ChatSession wenn remember=True
            if remember:
                await self.context_manager.add_interaction(
                    session_id,
                    'user',
                    query,
                    metadata={"user_id": user_id}
                )

            # Set user context variables
            timestamp = datetime.now()
            self.variable_manager.register_scope('user', {
                'id': user_id,
                'session': session_id,
                'query': query,
                'timestamp': timestamp.isoformat()
            })

            # Update system variables
            self.variable_manager.set('system_context.timestamp', {'isoformat': timestamp.isoformat()})
            self.variable_manager.set('system_context.current_session', session_id)
            self.variable_manager.set('system_context.current_user', user_id)
            self.variable_manager.set('system_context.last_query', query)

            # Initialize with tool awareness
            await self.initialize_context_awareness()

            # VEREINFACHT: Prepare execution context - weniger Daten duplizieren
            self.shared.update({
                "current_query": query,
                "session_id": session_id,
                "user_id": user_id,
                "stream_callback": stream_callback,
                "remember": remember,
                # CENTRAL: Context Manager ist die primäre Context-Quelle
                "context_manager": self.context_manager,
                "variable_manager": self.variable_manager
            })

            # Set LLM models in shared context
            self.shared['fast_llm_model'] = self.amd.fast_llm_model
            self.shared['complex_llm_model'] = self.amd.complex_llm_model
            self.shared['persona_config'] = self.amd.persona
            self.shared['use_fast_response'] = self.amd.use_fast_response

            # Set system status
            self.shared["system_status"] = "running"
            self.is_running = True

            # Execute main orchestration flow
            result = await self._orchestrate_execution()

            #Store assistant response in ChatSession wenn remember=True
            if remember:
                await self.context_manager.add_interaction(
                    session_id,
                    'assistant',
                    result,
                    metadata={"user_id": user_id, "execution_duration": time.time() - execution_start}
                )

            total_duration = self.progress_tracker.end_timer("total_execution")

            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="execution_complete",
                timestamp=time.time(),
                node_name="FlowAgent",
                status=NodeStatus.COMPLETED,
                node_duration=total_duration,
                session_id=session_id,
                metadata={
                    "result_length": len(result),
                    "summary": self.progress_tracker.get_summary(),
                    "remembered": remember
                }
            ))

            # Checkpoint if needed
            if self.enable_pause_resume:
                with Spinner("Creating checkpoint..."):
                    await self._maybe_checkpoint()
            return result

        except Exception as e:
            eprint(f"Agent execution failed: {e}", exc_info=True)
            error_response = f"I encountered an error: {str(e)}"
            result = error_response
            import traceback
            print(traceback.format_exc())

            # Store error in ChatSession wenn remember=True
            if remember:
                await self.context_manager.add_interaction(
                    session_id,
                    'assistant',
                    error_response,
                    metadata={
                        "user_id": user_id,
                        "error": True,
                        "error_type": type(e).__name__
                    }
                )

            total_duration = self.progress_tracker.end_timer("total_execution")

            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="error",
                timestamp=time.time(),
                node_name="FlowAgent",
                status=NodeStatus.FAILED,
                node_duration=total_duration,
                session_id=session_id,
                metadata={"error": str(e), "error_type": type(e).__name__}
            ))

            return error_response

        finally:
            self.shared["system_status"] = "idle"
            self.is_running = False
            self.active_session = None

    def set_response_format(
        self,
        response_format: str,
        text_length: str,
        custom_instructions: str = "",
        quality_threshold: float = 0.7
    ):
        """Dynamische Format- und Längen-Konfiguration"""

        # Validiere Eingaben
        try:
            ResponseFormat(response_format)
            TextLength(text_length)
        except ValueError:
            available_formats = [f.value for f in ResponseFormat]
            available_lengths = [l.value for l in TextLength]
            raise ValueError(
                f"Invalid format or length. "
                f"Available formats: {available_formats}. "
                f"Available lengths: {available_lengths}"
            )

        # Erstelle oder aktualisiere Persona
        if not self.amd.persona:
            self.amd.persona = PersonaConfig(name="Assistant")

        # Erstelle Format-Konfiguration
        format_config = FormatConfig(
            response_format=ResponseFormat(response_format),
            text_length=TextLength(text_length),
            custom_instructions=custom_instructions,
            quality_threshold=quality_threshold
        )

        self.amd.persona.format_config = format_config

        # Aktualisiere Personality Traits mit Format-Hinweisen
        self._update_persona_with_format(response_format, text_length)

        # Update shared state
        self.shared["persona_config"] = self.amd.persona
        self.shared["format_config"] = format_config

        rprint(f"Response format set: {response_format}, length: {text_length}")

    def _update_persona_with_format(self, response_format: str, text_length: str):
        """Aktualisiere Persona-Traits basierend auf Format"""

        # Format-spezifische Traits
        format_traits = {
            "with-tables": ["structured", "data-oriented", "analytical"],
            "with-bullet-points": ["organized", "clear", "systematic"],
            "with-lists": ["methodical", "sequential", "thorough"],
            "md-text": ["technical", "formatted", "detailed"],
            "yaml-text": ["structured", "machine-readable", "precise"],
            "json-text": ["technical", "API-focused", "structured"],
            "text-only": ["conversational", "natural", "flowing"],
            "pseudo-code": ["logical", "algorithmic", "step-by-step"],
            "code-structure": ["technical", "systematic", "hierarchical"]
        }

        # Längen-spezifische Traits
        length_traits = {
            "mini-chat": ["concise", "quick", "to-the-point"],
            "chat-conversation": ["conversational", "friendly", "balanced"],
            "table-conversation": ["structured", "comparative", "organized"],
            "detailed-indepth": ["thorough", "comprehensive", "analytical"],
            "phd-level": ["academic", "scholarly", "authoritative"]
        }

        # Kombiniere Traits
        current_traits = set(self.amd.persona.personality_traits)

        # Entferne alte Format-Traits
        old_format_traits = set()
        for traits in format_traits.values():
            old_format_traits.update(traits)
        for traits in length_traits.values():
            old_format_traits.update(traits)

        current_traits -= old_format_traits

        # Füge neue Traits hinzu
        new_traits = format_traits.get(response_format, [])
        new_traits.extend(length_traits.get(text_length, []))

        current_traits.update(new_traits)
        self.amd.persona.personality_traits = list(current_traits)

    def get_available_formats(self) -> dict[str, list[str]]:
        """Erhalte verfügbare Format- und Längen-Optionen"""
        return {
            "formats": [f.value for f in ResponseFormat],
            "lengths": [l.value for l in TextLength],
            "format_descriptions": {
                f.value: FormatConfig(response_format=f).get_format_instructions()
                for f in ResponseFormat
            },
            "length_descriptions": {
                l.value: FormatConfig(text_length=l).get_length_instructions()
                for l in TextLength
            }
        }

    async def a_run_with_format(
        self,
        query: str,
        response_format: str = "frei-text",
        text_length: str = "chat-conversation",
        custom_instructions: str = "",
        **kwargs
    ) -> str:
        """Führe Agent mit spezifischem Format aus"""

        # Temporäre Format-Einstellung
        original_persona = self.amd.persona

        try:
            self.set_response_format(response_format, text_length, custom_instructions)
            response = await self.a_run(query, **kwargs)
            return response
        finally:
            # Restore original persona
            self.amd.persona = original_persona
            self.shared["persona_config"] = original_persona

    def get_format_quality_report(self) -> dict[str, Any]:
        """Erhalte detaillierten Format-Qualitätsbericht"""
        quality_assessment = self.shared.get("quality_assessment", {})

        if not quality_assessment:
            return {"status": "no_assessment", "message": "No recent quality assessment available"}

        quality_details = quality_assessment.get("quality_details", {})

        return {
            "overall_score": quality_details.get("total_score", 0.0),
            "format_adherence": quality_details.get("format_adherence", 0.0),
            "length_adherence": quality_details.get("length_adherence", 0.0),
            "content_quality": quality_details.get("base_quality", 0.0),
            "llm_assessment": quality_details.get("llm_assessment", 0.0),
            "suggestions": quality_assessment.get("suggestions", []),
            "assessment": quality_assessment.get("quality_assessment", "unknown"),
            "format_config_active": quality_details.get("format_config_used", False)
        }

    def get_variable_documentation(self) -> str:
        """Get comprehensive variable system documentation"""
        docs = []
        docs.append("# Variable System Documentation\n")

        # Available scopes
        docs.append("## Available Scopes:")
        scope_info = self.variable_manager.get_scope_info()
        for scope_name, info in scope_info.items():
            docs.append(f"- `{scope_name}`: {info['type']} with {info.get('keys', 'N/A')} keys")

        docs.append("\n## Syntax Options:")
        docs.append("- `{{ variable.path }}` - Full path resolution")
        docs.append("- `{variable}` - Simple variable (no dots)")
        docs.append("- `$variable` - Shell-style variable")

        docs.append("\n## Example Usage:")
        docs.append("- `{{ results.task_1.data }}` - Get result from task_1")
        docs.append("- `{{ user.name }}` - Get user name")
        docs.append("- `{agent_name}` - Simple agent name")
        docs.append("- `$timestamp` - System timestamp")

        # Available variables
        docs.append("\n## Available Variables:")
        variables = self.variable_manager.get_available_variables()
        for scope_name, scope_vars in variables.items():
            docs.append(f"\n### {scope_name}:")
            for _var_name, var_info in scope_vars.items():
                docs.append(f"- `{var_info['path']}`: {var_info['preview']} ({var_info['type']})")

        return "\n".join(docs)

    def _setup_variable_scopes(self):
        """Setup default variable scopes with enhanced structure"""
        self.variable_manager.register_scope('agent', {
            'name': self.amd.name,
            'model_fast': self.amd.fast_llm_model,
            'model_complex': self.amd.complex_llm_model
        })

        timestamp = datetime.now()
        self.variable_manager.register_scope('system', {
            'timestamp': timestamp.isoformat(),
            'version': '2.0',
            'capabilities': list(self._tool_capabilities.keys())
        })

        # ADDED: Initialize empty results and tasks scopes
        self.variable_manager.register_scope('results', {})
        self.variable_manager.register_scope('tasks', {})

        # Update shared state
        self.shared["variable_manager"] = self.variable_manager

    def set_variable(self, path: str, value: Any):
        """Set variable using unified system"""
        self.variable_manager.set(path, value)

    def get_variable(self, path: str, default=None):
        """Get variable using unified system"""
        return self.variable_manager.get(path, default)

    def format_text(self, text: str, **context) -> str:
        """Format text with variables"""
        return self.variable_manager.format_text(text, context)

    async def initialize_session_context(self, session_id: str = "default", max_history: int = 200) -> bool:
        """Vereinfachte Session-Initialisierung über UnifiedContextManager"""
        try:
            # Delegation an UnifiedContextManager
            session = await self.context_manager.initialize_session(session_id, max_history)

            # Ensure Variable Manager integration
            if not self.context_manager.variable_manager:
                self.context_manager.variable_manager = self.variable_manager

            # Update shared state (minimal - primary data now in context_manager)
            self.shared["active_session_id"] = session_id
            self.shared["session_initialized"] = True

            # Legacy support: Keep session_managers reference in shared for backward compatibility
            self.shared["session_managers"] = self.context_manager.session_managers

            rprint(f"Session context initialized for {session_id} via UnifiedContextManager")
            return True

        except Exception as e:
            eprint(f"Session context initialization failed: {e}")
            import traceback
            print(traceback.format_exc())
            return False

    async def initialize_context_awareness(self):
        """Enhanced context awareness with session management"""

        # Initialize session if not already done
        session_id = self.shared.get("session_id", self.active_session)
        if not self.shared.get("session_initialized"):
            await self.initialize_session_context(session_id)

        # Ensure tool capabilities are loaded
        # add tqdm prigress bar

        from tqdm import tqdm

        if hasattr(self.task_flow, 'llm_reasoner'):
            if "read_from_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_read_from_variables'):
                await self.add_tool(lambda scope, key, purpose: self.task_flow.llm_reasoner._execute_read_from_variables({"scope": scope, "key": key, "purpose": purpose}), "read_from_variables", "Read from variables")
            if "write_to_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_write_to_variables'):
                await self.add_tool(lambda scope, key, value, description: self.task_flow.llm_reasoner._execute_write_to_variables({"scope": scope, "key": key, "value": value, "description": description}), "write_to_variables", "Write to variables")

            if "internal_reasoning" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_internal_reasoning'):
                async def internal_reasoning_tool(thought:str, thought_number:int, total_thoughts:int, next_thought_needed:bool, current_focus:str, key_insights:list[str], potential_issues:list[str], confidence_level:float):
                    args = {
                        "thought": thought,
                        "thought_number": thought_number,
                        "total_thoughts": total_thoughts,
                        "next_thought_needed": next_thought_needed,
                        "current_focus": current_focus,
                        "key_insights": key_insights,
                        "potential_issues": potential_issues,
                        "confidence_level": confidence_level
                    }
                    return await self.task_flow.llm_reasoner._execute_internal_reasoning(args, self.shared)
                await self.add_tool(internal_reasoning_tool, "internal_reasoning", "Internal reasoning")

            if "manage_internal_task_stack" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_manage_task_stack'):
                async def manage_internal_task_stack_tool(action:str, task_description:str, outline_step_ref:str):
                    args = {
                        "action": action,
                        "task_description": task_description,
                        "outline_step_ref": outline_step_ref
                    }
                    return await self.task_flow.llm_reasoner._execute_manage_task_stack(args, self.shared)
                await self.add_tool(manage_internal_task_stack_tool, "manage_internal_task_stack", "Manage internal task stack")

            if "outline_step_completion" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_outline_step_completion'):
                async def outline_step_completion_tool(step_completed:bool, completion_evidence:str, next_step_focus:str):
                    args = {
                        "step_completed": step_completed,
                        "completion_evidence": completion_evidence,
                        "next_step_focus": next_step_focus
                    }
                    return await self.task_flow.llm_reasoner._execute_outline_step_completion(args, self.shared)
                await self.add_tool(outline_step_completion_tool, "outline_step_completion", "Outline step completion")


        registered_tools = set(self._tool_registry.keys())
        cached_capabilities = list(self._tool_capabilities.keys())  # Create a copy of
        for tool_name in cached_capabilities:
            if tool_name in self._tool_capabilities and tool_name not in registered_tools:
                del self._tool_capabilities[tool_name]
                iprint(f"Removed outdated capability for unavailable tool: {tool_name}")

        for tool_name in tqdm(self.shared["available_tools"], desc=f"Agent {self.amd.name} Analyzing Tools", unit="tool", colour="green", total=len(self.shared["available_tools"])):
            if tool_name not in self._tool_capabilities:
                tool_info = self._tool_registry.get(tool_name, {})
                description = tool_info.get("description", "No description")
                with Spinner(f"Analyzing tool {tool_name}"):
                    await self._analyze_tool_capabilities(tool_name, description, tool_info.get("args_schema", "()"))

            if tool_name in self._tool_capabilities:
                function = self._tool_registry[tool_name]["function"]
                if not isinstance(self._tool_capabilities[tool_name], dict):
                    self._tool_capabilities[tool_name] = {}
                self._tool_capabilities[tool_name]["args_schema"] = get_args_schema(function)

        # Set enhanced system context
        self.shared["system_context"] = {
            "capabilities_summary": self._build_capabilities_summary(),
            "tool_count": len(self.shared["available_tools"]),
            "analysis_loaded": len(self._tool_capabilities),
            "intelligence_level": "high" if self._tool_capabilities else "basic",
            "context_management": "advanced_session_aware",
            "session_managers": len(self.shared.get("session_managers", {})),
        }


        rprint("Advanced context awareness initialized with session management")

    async def get_context(self, session_id: str = None, format_for_llm: bool = True) -> str | dict[str, Any]:
        """
        ÜBERARBEITET: Get context über UnifiedContextManager statt verteilte Quellen
        """
        try:
            session_id = session_id or self.shared.get("session_id", self.active_session)
            query = self.shared.get("current_query", "")

            #Hole unified context über Context Manager
            unified_context = await self.context_manager.build_unified_context(session_id, query, "full")


            if format_for_llm:
                return self.context_manager.get_formatted_context_for_llm(unified_context)
            else:
                return unified_context

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            eprint(f"Failed to generate context via UnifiedContextManager: {e}")

            # FALLBACK: Fallback zu alter Methode falls UnifiedContextManager fehlschlägt
            if format_for_llm:
                return f"Error generating context: {str(e)}"
            else:
                return {
                    "error": str(e),
                    "generated_at": datetime.now().isoformat(),
                    "fallback_mode": True
                }

    def get_context_statistics(self) -> dict[str, Any]:
        """Get comprehensive context management statistics"""
        stats = {
            "context_system": "advanced_session_aware",
            "compression_threshold": 0.76,
            "max_tokens": getattr(self, 'max_input_tokens', 8000),
            "session_managers": {},
            "context_usage": {},
            "compression_stats": {}
        }

        # Session manager statistics
        session_managers = self.shared.get("session_managers", {})
        for name, manager in session_managers.items():
            stats["session_managers"][name] = {
                "history_length": len(manager.history if hasattr(manager, 'history') else (manager.get("history", []) if hasattr(manager, 'get') else [])),
                "max_length": manager.max_length if hasattr(manager, 'max_length') else manager.get("max_length", 0),
                "space_name": manager.space_name if hasattr(manager, 'space_name') else manager.get("space_name", "")
            }

        # Context node statistics if available
        if hasattr(self.task_flow, 'context_manager'):
            context_manager = self.task_flow.context_manager
            stats["compression_stats"] = {
                "compression_threshold": context_manager.compression_threshold,
                "max_tokens": context_manager.max_tokens,
                "active_sessions": len(context_manager.session_managers)
            }

        # LLM call statistics from enhanced node
        llm_stats = self.shared.get("llm_call_stats", {})
        if llm_stats:
            stats["context_usage"] = {
                "total_llm_calls": llm_stats.get("total_calls", 0),
                "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
                "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                        1)
            }

        return stats

    def set_persona(self, name: str, style: str = "professional", tone: str = "friendly",
                    personality_traits: list[str] = None, apply_method: str = "system_prompt",
                    integration_level: str = "light", custom_instructions: str = ""):
        """Set agent persona mit erweiterten Konfigurationsmöglichkeiten"""
        if personality_traits is None:
            personality_traits = ["helpful", "concise"]

        self.amd.persona = PersonaConfig(
            name=name,
            style=style,
            tone=tone,
            personality_traits=personality_traits,
            custom_instructions=custom_instructions,
            apply_method=apply_method,
            integration_level=integration_level
        )

        rprint(f"Persona set: {name} ({style}, {tone}) - Method: {apply_method}, Level: {integration_level}")

    def configure_persona_integration(self, apply_method: str = "system_prompt", integration_level: str = "light"):
        """Configure how persona is applied"""
        if self.amd.persona:
            self.amd.persona.apply_method = apply_method
            self.amd.persona.integration_level = integration_level
            rprint(f"Persona integration updated: {apply_method}, {integration_level}")
        else:
            wprint("No persona configured to update")

    def get_available_variables(self) -> dict[str, str]:
        """Get available variables for dynamic formatting"""
        return self.variable_manager.get_available_variables()

    async def _orchestrate_execution(self) -> str:
        """
        Enhanced orchestration with LLMReasonerNode as strategic core.
        The reasoner now handles both task management and response generation internally.
        """

        self.shared["agent_instance"] = self
        self.shared["session_id"] = self.active_session
        # === UNIFIED REASONING AND EXECUTION CYCLE ===
        rprint("Starting strategic reasoning and execution cycle")

        # The LLMReasonerNode now handles the complete cycle:
        # 1. Strategic analysis of the query
        # 2. Decision making about approach
        # 3. Orchestration of sub-systems (LLMToolNode, TaskPlanner/Executor)
        # 4. Response synthesis and formatting

        # Execute the unified flow
        task_management_result = await self.task_flow.run_async(self.shared)

        # Check for various completion states
        if self.shared.get("plan_halted"):
            error_response = f"Task execution was halted: {self.shared.get('halt_reason', 'Unknown reason')}"
            self.shared["current_response"] = error_response
            return error_response

        final_response = self.shared.get("current_response", "Task completed successfully.")
        # Execute ResponseGenerationFlow for persona application and formatting
        response_result = await self.response_flow.run_async(self.shared)

        # The reasoner provides the final response
        final_response = self.shared.get("current_response", "Task completed successfully.")

        # Add reasoning artifacts to response if available
        reasoning_artifacts = self.shared.get("reasoning_artifacts", {})
        if reasoning_artifacts and reasoning_artifacts.get("reasoning_loops", 0) > 1:
            # For debugging/transparency, could add reasoning info to metadata
            pass

        # Log enhanced statistics
        self._log_execution_stats()

        return final_response

    def _log_execution_stats(self):
        """Enhanced execution statistics with reasoning metrics"""
        tasks = self.shared.get("tasks", {})
        adaptations = self.shared.get("plan_adaptations", 0)
        reasoning_artifacts = self.shared.get("reasoning_artifacts", {})

        completed_tasks = sum(1 for t in tasks.values() if t.status == "completed")
        failed_tasks = sum(1 for t in tasks.values() if t.status == "failed")

        # Enhanced logging with reasoning metrics
        reasoning_loops = reasoning_artifacts.get("reasoning_loops", 0)

        stats_message = f"Execution complete - Tasks: {completed_tasks} completed, {failed_tasks} failed"

        if adaptations > 0:
            stats_message += f", {adaptations} adaptations"

        if reasoning_loops > 0:
            stats_message += f", {reasoning_loops} reasoning loops"

            # Add reasoning efficiency metric
            if completed_tasks > 0:
                efficiency = completed_tasks / max(reasoning_loops, 1)
                stats_message += f" (efficiency: {efficiency:.1f} tasks/loop)"

        rprint(stats_message)

        # Log reasoning context if significant
        if reasoning_loops > 3:
            internal_task_stack = reasoning_artifacts.get("internal_task_stack", [])
            completed_reasoning_tasks = len([t for t in internal_task_stack if t.get("status") == "completed"])

            if completed_reasoning_tasks > 0:
                rprint(f"Strategic reasoning: {completed_reasoning_tasks} high-level tasks completed")

    def _build_capabilities_summary(self) -> str:
        """Build summary of agent capabilities"""

        if not self._tool_capabilities:
            return "Basic LLM capabilities only"

        summaries = []
        for tool_name, cap in self._tool_capabilities.items():
            primary = cap.get('primary_function', 'Unknown function')
            summaries.append(f"{tool_name}{cap.get('args_schema', '()')}: {primary}")

        return f"Enhanced capabilities: {'; '.join(summaries)}"

    # Neue Hilfsmethoden für erweiterte Funktionalität

    async def get_task_execution_summary(self) -> dict[str, Any]:
        """Erhalte detaillierte Zusammenfassung der Task-Ausführung"""
        tasks = self.shared.get("tasks", {})
        results_store = self.shared.get("results", {})

        summary = {
            "total_tasks": len(tasks),
            "completed_tasks": [],
            "failed_tasks": [],
            "task_types_used": {},
            "tools_used": [],
            "adaptations": self.shared.get("plan_adaptations", 0),
            "execution_timeline": []
        }

        for task_id, task in tasks.items():
            task_info = {
                "id": task_id,
                "type": task.type,
                "description": task.description,
                "status": task.status,
                "duration": None
            }

            if task.started_at and task.completed_at:
                duration = (task.completed_at - task.started_at).total_seconds()
                task_info["duration"] = duration

            if task.status == "completed":
                summary["completed_tasks"].append(task_info)
                if isinstance(task, ToolTask):
                    summary["tools_used"].append(task.tool_name)
            elif task.status == "failed":
                task_info["error"] = task.error
                summary["failed_tasks"].append(task_info)

            # Task types counting
            task_type = task.type
            summary["task_types_used"][task_type] = summary["task_types_used"].get(task_type, 0) + 1

        return summary

    async def explain_reasoning_process(self) -> str:
        """Erkläre den Reasoning-Prozess des Agenten"""
        if not LITELLM_AVAILABLE:
            return "Reasoning explanation requires LLM capabilities."

        summary = await self.get_task_execution_summary()

        prompt = f"""
Erkläre den Reasoning-Prozess dieses AI-Agenten in verständlicher Form:

## Ausführungszusammenfassung
- Total Tasks: {summary['total_tasks']}
- Erfolgreich: {len(summary['completed_tasks'])}
- Fehlgeschlagen: {len(summary['failed_tasks'])}
- Plan-Adaptationen: {summary['adaptations']}
- Verwendete Tools: {', '.join(set(summary['tools_used']))}
- Task-Typen: {summary['task_types_used']}

## Task-Details
Erfolgreiche Tasks:
{self._format_tasks_for_explanation(summary['completed_tasks'])}

## Anweisungen
Erkläre in 2-3 Absätzen:
1. Welche Strategie der Agent gewählt hat
2. Wie er die Aufgabe in Tasks unterteilt hat
3. Wie er auf unerwartete Ergebnisse reagiert hat (falls Adaptationen)
4. Was die wichtigsten Erkenntnisse waren

Schreibe für einen technischen Nutzer, aber verständlich."""

        try:
            response = await self.a_run_llm_completion(
                model=self.amd.complex_llm_model,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.5,
                max_tokens=800,task_id="reasoning_explanation"
            )

            return response

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            return f"Could not generate reasoning explanation: {e}"

    def _format_tasks_for_explanation(self, tasks: list[dict]) -> str:
        formatted = []
        for task in tasks[:5]:  # Top 5 tasks
            duration_info = f" ({task['duration']:.1f}s)" if task['duration'] else ""
            formatted.append(f"- {task['type']}: {task['description']}{duration_info}")
        return "\n".join(formatted)

    # ===== PAUSE/RESUME FUNCTIONALITY =====

    async def pause(self) -> bool:
        """Pause agent execution"""
        if not self.is_running:
            return False

        self.is_paused = True
        self.shared["system_status"] = "paused"

        # Create checkpoint
        checkpoint = await self._create_checkpoint()
        await self._save_checkpoint(checkpoint)

        rprint("Agent execution paused")
        return True

    async def resume(self) -> bool:
        """Resume agent execution"""
        if not self.is_paused:
            return False

        self.is_paused = False
        self.shared["system_status"] = "running"

        rprint("Agent execution resumed")
        return True

    # ===== CHECKPOINT MANAGEMENT =====

    async def _create_checkpoint(self) -> AgentCheckpoint:
        """Vereinfachte Checkpoint-Erstellung - fokussiert auf wesentliche Daten"""
        try:
            # Budget Manager Daten vor Checkpoint speichern
            if hasattr(self.amd, 'budget_manager') and self.amd.budget_manager:
                self.amd.budget_manager.save_data()

            # Bereite AMD-Daten vor (ohne budget_manager für Serialisierung)
            amd_data = self.amd.model_dump()
            amd_data['budget_manager'] = None

            # Sammle wesentliche Session-Daten (vereinfacht)
            session_data = {}
            if self.context_manager and self.context_manager.session_managers:
                for session_id, session in self.context_manager.session_managers.items():
                    try:
                        if hasattr(session, 'history') and session.history:
                            # Nur die letzten 20 Nachrichten pro Session für Checkpoint
                            recent_history = session.history[-20:]
                            session_data[session_id] = {
                                "history": recent_history,
                                "session_type": "chatsession",
                                "message_count": len(session.history)
                            }
                        elif isinstance(session, dict) and session.get('history'):
                            session_data[session_id] = {
                                "history": session['history'][-20:],
                                "session_type": "fallback",
                                "message_count": len(session['history'])
                            }
                    except Exception as e:
                        rprint(f"Skipping session {session_id} in checkpoint: {e}")

            # Sammle serialisierbare Variable-Scopes
            variable_scopes = {}
            if self.variable_manager:
                NON_SERIALIZABLE_KEYS = {
                    "tool_registry", "variable_manager", "context_manager", "agent_instance",
                    "llm_tool_node_instance", "task_planner_instance", "task_executor_instance",
                    "progress_tracker", "session_managers", "stream_callback"
                }

                for scope_name, scope_data in self.variable_manager.scopes.items():
                    if isinstance(scope_data, dict):
                        # Filtere nicht-serialisierbare Objekte heraus
                        clean_scope = {
                            k: v for k, v in scope_data.items()
                            if k not in NON_SERIALIZABLE_KEYS
                        }
                        variable_scopes[scope_name] = clean_scope
                    else:
                        try:
                            # Teste Serialisierbarkeit
                            pickle.dumps(scope_data)
                            variable_scopes[scope_name] = scope_data
                        except:
                            # Überspringe nicht-serialisierbare Scopes
                            pass

            # Erstelle konsolidierten Checkpoint
            checkpoint = AgentCheckpoint(
                timestamp=datetime.now(),
                agent_state={
                    "is_running": self.is_running,
                    "is_paused": self.is_paused,
                    "amd_data": amd_data,
                    "active_session": self.active_session,
                    "system_status": self.shared.get("system_status", "idle")
                },
                task_state={
                    task_id: asdict(task) for task_id, task in self.shared.get("tasks", {}).items()
                },
                world_model=self.shared.get("world_model", {}),
                active_flows=["task_flow", "response_flow"],
                metadata={
                    "session_id": self.shared.get("session_id", "default"),
                    "last_query": self.shared.get("current_query", ""),
                    "checkpoint_version": "3.0_simplified",
                    "agent_name": self.amd.name
                },
                # Konsolidierte Zusatzdaten
                session_data=session_data,
                variable_scopes=variable_scopes,
                results_store=self.shared.get("results", {}),
                conversation_history=self.shared.get("conversation_history", [])[-50:],  # Letzte 50 Nachrichten
                tool_capabilities=self._tool_capabilities.copy()
            )

            rprint(f"Vereinfachter Checkpoint erstellt mit {len(session_data)} Sessions")
            return checkpoint

        except Exception as e:
            eprint(f"Checkpoint-Erstellung fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            raise

    async def _save_checkpoint(self, checkpoint: AgentCheckpoint, filepath: str = None):
        """Vereinfachtes Checkpoint-Speichern - alles in eine Datei"""
        try:
            from toolboxv2 import get_app
            folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name
            if not os.path.exists(folder):
                os.makedirs(folder, exist_ok=True)

            if not filepath:
                timestamp = checkpoint.timestamp.strftime("%Y%m%d_%H%M%S")
                filepath = f"agent_checkpoint_{timestamp}.pkl"
            filepath = os.path.join(folder, filepath)

            # Sessions vor dem Speichern synchronisieren
            if self.context_manager and self.context_manager.session_managers:
                for session_id, session in self.context_manager.session_managers.items():
                    try:
                        if hasattr(session, 'save'):
                            await session.save()
                        elif hasattr(session, '_save_to_memory'):
                            session._save_to_memory()
                    except Exception as e:
                        rprint(f"Session sync error für {session_id}: {e}")

            # Speichere Checkpoint direkt
            with open(filepath, 'wb') as f:
                pickle.dump(checkpoint, f)

            self.last_checkpoint = checkpoint.timestamp

            # Erstelle einfache Zusammenfassung
            summary_parts = []
            if hasattr(checkpoint, 'session_data') and checkpoint.session_data:
                summary_parts.append(f"{len(checkpoint.session_data)} sessions")
            if checkpoint.task_state:
                completed_tasks = len([t for t in checkpoint.task_state.values() if t.get("status") == "completed"])
                summary_parts.append(f"{completed_tasks} completed tasks")
            if hasattr(checkpoint, 'variable_scopes') and checkpoint.variable_scopes:
                summary_parts.append(f"{len(checkpoint.variable_scopes)} variable scopes")

            summary = "; ".join(summary_parts) if summary_parts else "Basic checkpoint"
            rprint(f"Checkpoint gespeichert: {filepath} ({summary})")
            return True

        except Exception as e:
            eprint(f"Checkpoint-Speicherung fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            return False

    async def load_latest_checkpoint(self, auto_restore_history: bool = True, max_age_hours: int = 24) -> dict[
        str, Any]:
        """Vereinfachtes Checkpoint-Laden mit automatischer History-Wiederherstellung"""
        try:
            from toolboxv2 import get_app
            folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

            if not os.path.exists(folder):
                return {"success": False, "error": "Kein Checkpoint-Verzeichnis gefunden"}

            # Finde neuesten Checkpoint
            checkpoint_files = []
            for file in os.listdir(folder):
                if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                    filepath = os.path.join(folder, file)
                    try:
                        timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                        if timestamp_str == 'final_checkpoint':
                            file_time = datetime.fromtimestamp(os.path.getmtime(filepath))
                        else:
                            file_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")

                        age_hours = (datetime.now() - file_time).total_seconds() / 3600
                        if age_hours <= max_age_hours:
                            checkpoint_files.append((filepath, file_time, age_hours))
                    except Exception:
                        continue

            if not checkpoint_files:
                return {"success": False, "error": f"Keine gültigen Checkpoints in {max_age_hours} Stunden gefunden"}

            # Lade neuesten Checkpoint
            checkpoint_files.sort(key=lambda x: x[1], reverse=True)
            latest_checkpoint_path, latest_timestamp, age_hours = checkpoint_files[0]

            rprint(f"Lade Checkpoint: {latest_checkpoint_path} (Alter: {age_hours:.1f}h)")

            with open(latest_checkpoint_path, 'rb') as f:
                checkpoint: AgentCheckpoint = pickle.load(f)

            # Stelle Agent-Status wieder her
            restore_stats = await self._restore_from_checkpoint_simplified(checkpoint, auto_restore_history)

            # Re-initialisiere Kontext-Awareness
            await self.initialize_context_awareness()

            return {
                "success": True,
                "checkpoint_file": latest_checkpoint_path,
                "checkpoint_age_hours": age_hours,
                "checkpoint_timestamp": latest_timestamp.isoformat(),
                "available_checkpoints": len(checkpoint_files),
                "restore_stats": restore_stats
            }

        except Exception as e:
            eprint(f"Checkpoint-Laden fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            return {"success": False, "error": str(e)}

    async def _restore_from_checkpoint_simplified(self, checkpoint: AgentCheckpoint, auto_restore_history: bool) -> \
    dict[
        str, Any]:
        """Vereinfachte Checkpoint-Wiederherstellung"""
        restore_stats = {
            "agent_state_restored": False,
            "world_model_restored": False,
            "tasks_restored": 0,
            "sessions_restored": 0,
            "variables_restored": 0,
            "conversation_restored": 0,
            "errors": []
        }

        try:
            # 1. Agent-Status wiederherstellen
            if checkpoint.agent_state:
                self.is_running = checkpoint.agent_state.get("is_running", False)
                self.is_paused = checkpoint.agent_state.get("is_paused", False)
                self.active_session = checkpoint.agent_state.get("active_session")

                # AMD-Daten selektiv wiederherstellen
                amd_data = checkpoint.agent_state.get("amd_data", {})
                if amd_data:
                    # Nur sichere Felder wiederherstellen
                    safe_fields = ["name", "use_fast_response", "max_input_tokens"]
                    for field in safe_fields:
                        if field in amd_data and hasattr(self.amd, field):
                            setattr(self.amd, field, amd_data[field])

                    # Persona wiederherstellen falls vorhanden
                    if "persona" in amd_data and amd_data["persona"]:
                        try:
                            persona_data = amd_data["persona"]
                            if isinstance(persona_data, dict):
                                self.amd.persona = PersonaConfig(**persona_data)
                        except Exception as e:
                            restore_stats["errors"].append(f"Persona restore failed: {e}")

                restore_stats["agent_state_restored"] = True

            # 2. World Model wiederherstellen
            if checkpoint.world_model:
                self.shared["world_model"] = checkpoint.world_model.copy()
                self.world_model = checkpoint.world_model.copy()
                restore_stats["world_model_restored"] = True

            # 3. Variable System wiederherstellen
            if hasattr(checkpoint, 'variable_scopes') and checkpoint.variable_scopes:
                # Variable Manager neu initialisieren
                self.variable_manager = VariableManager(self.shared["world_model"], self.shared)

                # Basis-Scopes einrichten
                self._setup_variable_scopes()

                # Gespeicherte Scopes wiederherstellen
                for scope_name, scope_data in checkpoint.variable_scopes.items():
                    try:
                        self.variable_manager.register_scope(scope_name, scope_data)
                        restore_stats["variables_restored"] += 1
                    except Exception as e:
                        restore_stats["errors"].append(f"Variable scope {scope_name}: {e}")

                # Runtime-Objekte wieder einsetzen
                self.variable_manager.set("shared", "variable_manager", self.variable_manager)
                self.variable_manager.set("shared", "context_manager", self.context_manager)
                self.variable_manager.set("shared", "agent_instance", self)

                self.shared["variable_manager"] = self.variable_manager

            # 4. Tasks wiederherstellen
            if checkpoint.task_state:
                restored_tasks = {}
                for task_id, task_data in checkpoint.task_state.items():
                    try:
                        task_type = task_data.get("type", "generic")
                        if task_type == "LLMTask":
                            restored_tasks[task_id] = LLMTask(**task_data)
                        elif task_type == "ToolTask":
                            restored_tasks[task_id] = ToolTask(**task_data)
                        elif task_type == "DecisionTask":
                            restored_tasks[task_id] = DecisionTask(**task_data)
                        else:
                            restored_tasks[task_id] = Task(**task_data)

                        restore_stats["tasks_restored"] += 1
                    except Exception as e:
                        restore_stats["errors"].append(f"Task {task_id}: {e}")

                self.shared["tasks"] = restored_tasks

            # 5. Results Store wiederherstellen
            if hasattr(checkpoint, 'results_store') and checkpoint.results_store:
                self.shared["results"] = checkpoint.results_store
                if self.variable_manager:
                    self.variable_manager.set_results_store(checkpoint.results_store)

            # 6. Sessions und Conversation wiederherstellen (falls gewünscht)
            if auto_restore_history:
                await self._restore_sessions_and_conversation_simplified(checkpoint, restore_stats)

            # 7. Tool Capabilities wiederherstellen
            if hasattr(checkpoint, 'tool_capabilities') and checkpoint.tool_capabilities:
                self._tool_capabilities = checkpoint.tool_capabilities.copy()

            # Status setzen
            self.shared["system_status"] = "restored"
            restore_stats["restoration_timestamp"] = datetime.now().isoformat()

            rprint(
                f"Checkpoint wiederhergestellt: {restore_stats['tasks_restored']} Tasks, {restore_stats['sessions_restored']} Sessions, {len(restore_stats['errors'])} Fehler")
            return restore_stats

        except Exception as e:
            eprint(f"Checkpoint-Wiederherstellung fehlgeschlagen: {e}")
            import traceback
            print(traceback.format_exc())
            restore_stats["errors"].append(f"Critical restore error: {e}")
            return restore_stats

    async def _restore_sessions_and_conversation_simplified(self, checkpoint: AgentCheckpoint, restore_stats: dict):
        """Vereinfachte Session- und Conversation-Wiederherstellung"""
        try:
            # Context Manager sicherstellen
            if not self.context_manager:
                self.context_manager = UnifiedContextManager(self)
                self.context_manager.variable_manager = self.variable_manager

            # Sessions wiederherstellen
            if hasattr(checkpoint, 'session_data') and checkpoint.session_data:
                for session_id, session_info in checkpoint.session_data.items():
                    try:
                        # Session über Context Manager initialisieren
                        max_length = session_info.get("message_count", 200)
                        restored_session = await self.context_manager.initialize_session(session_id, max_length)

                        # History wiederherstellen
                        history = session_info.get("history", [])
                        if history and hasattr(restored_session, 'history'):
                            # Direkt in Session-History einfügen
                            restored_session.history.extend(history)

                        restore_stats["sessions_restored"] += 1
                    except Exception as e:
                        restore_stats["errors"].append(f"Session {session_id}: {e}")

            # Conversation History wiederherstellen
            if hasattr(checkpoint, 'conversation_history') and checkpoint.conversation_history:
                self.shared["conversation_history"] = checkpoint.conversation_history
                restore_stats["conversation_restored"] = len(checkpoint.conversation_history)

            # Update shared context
            self.shared["context_manager"] = self.context_manager
            if self.context_manager.session_managers:
                self.shared["session_managers"] = self.context_manager.session_managers
                self.shared["session_initialized"] = True

        except Exception as e:
            restore_stats["errors"].append(f"Session/conversation restore failed: {e}")

    async def _maybe_checkpoint(self):
        """Vereinfachtes automatisches Checkpointing"""
        if not self.enable_pause_resume:
            return

        now = datetime.now()
        if (not self.last_checkpoint or
            (now - self.last_checkpoint).seconds >= self.checkpoint_interval):

            try:
                checkpoint = await self._create_checkpoint()
                await self._save_checkpoint(checkpoint)
            except Exception as e:
                eprint(f"Automatic checkpoint failed: {e}")


    async def save_context_to_session(self, session_id: str = None, context_type: str = "full") -> bool:
        """Save current context to ChatSession for persistent storage"""
        try:
            session_id = session_id or self.shared.get("session_id", "default")

            if not self.context_manager:
                eprint("Context manager not available")
                return False

            # Build comprehensive context
            unified_context = await self.context_manager.build_unified_context(session_id, None, context_type)

            # Create context message for session storage
            context_message = {
                "role": "system",
                "content": f"[CONTEXT_SNAPSHOT_{context_type.upper()}] " + json.dumps(unified_context, default=str),
                "timestamp": datetime.now().isoformat(),
                "context_type": context_type,
                "metadata": {
                    "is_context_snapshot": True,
                    "context_version": "2.0",
                    "agent_name": self.amd.name,
                    "session_stats": unified_context.get("session_stats", {}),
                    "variables_count": len(unified_context.get("variables", {}).get("recent_results", [])),
                    "execution_state": unified_context.get("execution_state", {}).get("system_status", "unknown")
                }
            }

            # Store in session
            await self.context_manager.add_interaction(
                session_id,
                "system",
                context_message["content"],
                metadata=context_message["metadata"]
            )

            rprint(f"Context snapshot saved to session {session_id} (type: {context_type})")
            return True

        except Exception as e:
            eprint(f"Failed to save context to session: {e}")
            return False

    async def load_context_from_session(self, session_id: str, context_type: str = "full") -> dict[str, Any]:
        """Load context from ChatSession storage"""
        try:
            if not self.context_manager:
                return {"error": "Context manager not available"}

            session = self.context_manager.session_managers.get(session_id)
            if not session:
                return {"error": f"Session {session_id} not found"}

            # Search for context snapshots in session history
            context_snapshots = []

            if hasattr(session, 'history'):
                for message in reversed(session.history):  # Search from newest
                    if (message.get("role") == "system" and
                        message.get("metadata", {}).get("is_context_snapshot") and
                        message.get("metadata", {}).get("context_type") == context_type):

                        try:
                            # Extract context data
                            content = message.get("content", "")
                            if content.startswith(f"[CONTEXT_SNAPSHOT_{context_type.upper()}]"):
                                json_data = content.replace(f"[CONTEXT_SNAPSHOT_{context_type.upper()}] ", "")
                                context_data = json.loads(json_data)
                                context_snapshots.append({
                                    "context": context_data,
                                    "timestamp": message.get("timestamp"),
                                    "metadata": message.get("metadata", {})
                                })
                        except Exception as e:
                            wprint(f"Failed to parse context snapshot: {e}")

            if context_snapshots:
                # Return most recent context snapshot
                latest_context = context_snapshots[0]
                rprint(f"Loaded context snapshot from session {session_id} (timestamp: {latest_context['timestamp']})")
                return latest_context["context"]
            else:
                return {"error": f"No context snapshots of type '{context_type}' found in session {session_id}"}

        except Exception as e:
            eprint(f"Failed to load context from session: {e}")
            return {"error": str(e)}

    async def cleanup_session_context(self, session_id: str = None, keep_count: int = 100,
                                      remove_old_snapshots: bool = True) -> dict[str, Any]:
        """Cleanup session context by removing old snapshots and entries"""
        try:
            session_id = session_id or self.shared.get("session_id", "default")

            if not self.context_manager:
                return {"error": "Context manager not available"}

            session = self.context_manager.session_managers.get(session_id)
            if not session or not hasattr(session, 'history'):
                return {"error": f"Session {session_id} not found or has no history"}

            cleanup_stats = {
                "original_message_count": len(session.history),
                "context_snapshots_removed": 0,
                "context_entries_removed": 0,
                "regular_messages_kept": 0,
                "cleanup_performed": False
            }

            if len(session.history) <= keep_count:
                return {**cleanup_stats, "message": "No cleanup needed"}

            # Separate different types of messages
            regular_messages = []
            context_snapshots = []
            context_entries = []

            for message in session.history:
                metadata = message.get("metadata", {})

                if metadata.get("is_context_snapshot"):
                    context_snapshots.append(message)
                elif metadata.get("is_context_entry"):
                    context_entries.append(message)
                else:
                    regular_messages.append(message)

            # Keep most recent regular messages
            messages_to_keep = regular_messages[-keep_count:]
            cleanup_stats["regular_messages_kept"] = len(messages_to_keep)

            # Keep most recent context snapshots (if not removing)
            if not remove_old_snapshots:
                recent_snapshots = context_snapshots[-5:]  # Keep last 5 snapshots
                messages_to_keep.extend(recent_snapshots)
            else:
                cleanup_stats["context_snapshots_removed"] = len(context_snapshots)

            # Keep persistent context entries
            persistent_entries = [
                entry for entry in context_entries
                if entry.get("persistent", True)
            ]
            messages_to_keep.extend(persistent_entries)
            cleanup_stats["context_entries_removed"] = len(context_entries) - len(persistent_entries)

            # Sort by timestamp and update session
            messages_to_keep.sort(key=lambda x: x.get("timestamp", ""))
            session.history = messages_to_keep

            cleanup_stats.update({
                "final_message_count": len(session.history),
                "cleanup_performed": True,
                "messages_removed": cleanup_stats["original_message_count"] - len(session.history)
            })

            rprint(f"Session cleanup completed: {cleanup_stats['messages_removed']} messages removed")
            return cleanup_stats

        except Exception as e:
            eprint(f"Failed to cleanup session context: {e}")
            return {"error": str(e)}

    def get_session_storage_stats(self) -> dict[str, Any]:
        """Get comprehensive session storage statistics"""
        try:
            stats = {
                "context_manager_active": bool(self.context_manager),
                "total_sessions": 0,
                "session_details": {},
                "storage_summary": {
                    "total_messages": 0,
                    "context_snapshots": 0,
                    "context_entries": 0,
                    "regular_messages": 0
                }
            }

            if not self.context_manager:
                return stats

            stats["total_sessions"] = len(self.context_manager.session_managers)

            for session_id, session in self.context_manager.session_managers.items():
                session_stats = {
                    "session_type": "chatsession" if hasattr(session, 'history') else "fallback",
                    "message_count": 0,
                    "context_snapshots": 0,
                    "context_entries": 0,
                    "regular_messages": 0,
                    "storage_size_estimate": 0
                }

                if hasattr(session, 'history'):
                    session_stats["message_count"] = len(session.history)

                    for message in session.history:
                        content_size = len(str(message))
                        session_stats["storage_size_estimate"] += content_size

                        metadata = message.get("metadata", {})
                        if metadata.get("is_context_snapshot"):
                            session_stats["context_snapshots"] += 1
                        elif metadata.get("is_context_entry"):
                            session_stats["context_entries"] += 1
                        else:
                            session_stats["regular_messages"] += 1

                elif isinstance(session, dict) and 'history' in session:
                    session_stats["message_count"] = len(session['history'])
                    session_stats["regular_messages"] = len(session['history'])
                    session_stats["storage_size_estimate"] = sum(len(str(msg)) for msg in session['history'])

                stats["session_details"][session_id] = session_stats

                # Update totals
                stats["storage_summary"]["total_messages"] += session_stats["message_count"]
                stats["storage_summary"]["context_snapshots"] += session_stats["context_snapshots"]
                stats["storage_summary"]["context_entries"] += session_stats["context_entries"]
                stats["storage_summary"]["regular_messages"] += session_stats["regular_messages"]

            # Estimate total storage size
            stats["storage_summary"]["estimated_total_size_kb"] = sum(
                details["storage_size_estimate"] for details in stats["session_details"].values()
            ) / 1024

            return stats

        except Exception as e:
            eprint(f"Failed to get session storage stats: {e}")
            return {"error": str(e)}

    def list_available_checkpoints(self, max_age_hours: int = 168) -> list[dict[str, Any]]:  # Default 1 week
        """List all available checkpoints with metadata"""
        try:
            from toolboxv2 import get_app
            folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

            if not os.path.exists(folder):
                return []

            checkpoints = []
            for file in os.listdir(folder):
                if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                    filepath = os.path.join(folder, file)
                    try:
                        # Get file info
                        file_stat = os.stat(filepath)
                        file_size = file_stat.st_size
                        modified_time = datetime.fromtimestamp(file_stat.st_mtime)

                        # Extract timestamp from filename
                        timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                        if timestamp_str == 'final_checkpoint':
                            checkpoint_time = modified_time
                            checkpoint_type = "final"
                        else:
                            checkpoint_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")
                            checkpoint_type = "regular"

                        # Check age
                        age_hours = (datetime.now() - checkpoint_time).total_seconds() / 3600
                        if age_hours <= max_age_hours:

                            # Try to load checkpoint metadata without full loading
                            metadata = {}
                            try:
                                with open(filepath, 'rb') as f:
                                    checkpoint = pickle.load(f)
                                metadata = {
                                    "tasks_count": len(checkpoint.task_state) if checkpoint.task_state else 0,
                                    "world_model_entries": len(checkpoint.world_model) if checkpoint.world_model else 0,
                                    "session_id": checkpoint.metadata.get("session_id", "unknown") if hasattr(
                                        checkpoint, 'metadata') and checkpoint.metadata else "unknown",
                                    "last_query": checkpoint.metadata.get("last_query", "unknown")[:100] if hasattr(
                                        checkpoint, 'metadata') and checkpoint.metadata else "unknown"
                                }
                            except:
                                metadata = {"load_error": True}

                            checkpoints.append({
                                "filepath": filepath,
                                "filename": file,
                                "checkpoint_type": checkpoint_type,
                                "timestamp": checkpoint_time.isoformat(),
                                "age_hours": round(age_hours, 1),
                                "file_size_kb": round(file_size / 1024, 1),
                                "metadata": metadata
                            })

                    except Exception as e:
                        import traceback
                        print(traceback.format_exc())
                        wprint(f"Could not analyze checkpoint file {file}: {e}")
                        continue

            # Sort by timestamp (newest first)
            checkpoints.sort(key=lambda x: x["timestamp"], reverse=True)

            return checkpoints

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            eprint(f"Failed to list checkpoints: {e}")
            return []

    async def delete_old_checkpoints(self, keep_count: int = 5, max_age_hours: int = 168) -> dict[str, Any]:
        """Delete old checkpoints, keeping the most recent ones"""
        try:
            checkpoints = self.list_available_checkpoints(
                max_age_hours=max_age_hours * 2)  # Look further back for deletion

            deleted_count = 0
            deleted_size_kb = 0
            errors = []

            if len(checkpoints) > keep_count:
                # Keep the newest, delete the rest (except final checkpoint)
                to_delete = checkpoints[keep_count:]

                for checkpoint in to_delete:
                    if checkpoint["checkpoint_type"] != "final":  # Never delete final checkpoint
                        try:
                            os.remove(checkpoint["filepath"])
                            deleted_count += 1
                            deleted_size_kb += checkpoint["file_size_kb"]
                            rprint(f"Deleted old checkpoint: {checkpoint['filename']}")
                        except Exception as e:
                            import traceback
                            print(traceback.format_exc())
                            errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

            # Also delete checkpoints older than max_age_hours
            old_checkpoints = [cp for cp in checkpoints if
                               cp["age_hours"] > max_age_hours and cp["checkpoint_type"] != "final"]
            for checkpoint in old_checkpoints:
                if checkpoint not in checkpoints[keep_count:]:  # Don't double-delete
                    try:
                        os.remove(checkpoint["filepath"])
                        deleted_count += 1
                        deleted_size_kb += checkpoint["file_size_kb"]
                        rprint(f"Deleted aged checkpoint: {checkpoint['filename']}")
                    except Exception as e:
                        import traceback
                        print(traceback.format_exc())
                        errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

            return {
                "success": True,
                "deleted_count": deleted_count,
                "freed_space_kb": round(deleted_size_kb, 1),
                "remaining_checkpoints": len(checkpoints) - deleted_count,
                "errors": errors
            }

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            eprint(f"Failed to delete old checkpoints: {e}")
            return {
                "success": False,
                "error": str(e),
                "deleted_count": 0
            }

    # ===== TOOL AND NODE MANAGEMENT =====
    def _get_tool_analysis_path(self) -> str:
        """Get path for tool analysis cache"""
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/capabilities/' + self.amd.name
        os.makedirs(folder, exist_ok=True)
        return folder + '/tool_capabilities.json'

    def _get_context_path(self, session_id=None) -> str:
        """Get path for tool analysis cache"""
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/context/' + self.amd.name
        os.makedirs(folder, exist_ok=True)
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        session_suffix = f"_session_{session_id}" if session_id else ""
        filepath = f"agent_context_{self.amd.name}_{timestamp}{session_suffix}.json"
        return folder + f'/{filepath}'

    def add_first_class_tool(self, tool_func: Callable, name: str, description: str):
        """
        Add a first-class meta-tool that can be used by the LLMReasonerNode.
        These are different from regular tools - they control agent sub-systems.

        Args:
            tool_func: The function to register as a meta-tool
            name: Name of the meta-tool
            description: Description of when and how to use it
        """

        if not asyncio.iscoroutinefunction(tool_func):
            @wraps(tool_func)
            async def async_wrapper(*args, **kwargs):
                return await asyncio.to_thread(tool_func, *args, **kwargs)

            effective_func = async_wrapper
        else:
            effective_func = tool_func

        tool_name = name or effective_func.__name__
        tool_description = description or effective_func.__doc__ or "No description"

        # Validate the tool function
        if not callable(tool_func):
            raise ValueError("Tool function must be callable")

        # Register in the reasoner's meta-tool registry (if reasoner exists)
        if hasattr(self.task_flow, 'llm_reasoner'):
            if not hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
                self.task_flow.llm_reasoner.meta_tools_registry = {}

            self.task_flow.llm_reasoner.meta_tools_registry[tool_name] = {
                "function": effective_func,
                "description": tool_description,
                "args_schema": get_args_schema(tool_func)
            }

            rprint(f"First-class meta-tool added: {tool_name}")
        else:
            wprint("LLMReasonerNode not available for first-class tool registration")

    async def add_tool(self, tool_func: Callable, name: str = None, description: str = None, is_new=False):
        """Enhanced tool addition with intelligent analysis"""
        if not asyncio.iscoroutinefunction(tool_func):
            @wraps(tool_func)
            async def async_wrapper(*args, **kwargs):
                return await asyncio.to_thread(tool_func, *args, **kwargs)

            effective_func = async_wrapper
        else:
            effective_func = tool_func

        tool_name = name or effective_func.__name__
        tool_description = description or effective_func.__doc__ or "No description"

        # Store in registry
        self._tool_registry[tool_name] = {
            "function": effective_func,
            "description": tool_description,
            "args_schema": get_args_schema(tool_func)
        }

        # Add to available tools list
        if tool_name not in self.shared["available_tools"]:
            self.shared["available_tools"].append(tool_name)

        # Intelligent tool analysis
        if is_new:
            await self._analyze_tool_capabilities(tool_name, tool_description)

        rprint(f"Tool added with analysis: {tool_name}")

    async def _analyze_tool_capabilities(self, tool_name: str, description: str, tool_args:str):
        """Analyze tool capabilities with LLM for smart usage"""

        # Try to load existing analysis
        existing_analysis = self._load_tool_analysis()

        if tool_name in existing_analysis:
            try:
                # Validate cached data against the Pydantic model
                ToolAnalysis.model_validate(existing_analysis[tool_name])
                self._tool_capabilities[tool_name] = existing_analysis[tool_name]
                rprint(f"Loaded and validated cached analysis for {tool_name}")
            except ValidationError as e:
                wprint(f"Cached data for {tool_name} is invalid and will be regenerated: {e}")
                del self._tool_capabilities[tool_name]

        if not LITELLM_AVAILABLE:
            # Fallback analysis
            self._tool_capabilities[tool_name] = {
                "use_cases": [description],
                "triggers": [tool_name.lower().replace('_', ' ')],
                "complexity": "unknown",
                "confidence": 0.3
            }
            return

        # LLM-based intelligent analysis
        prompt = f"""
Analyze this tool and identify ALL possible use cases, triggers, and connections:

Tool Name: {tool_name}
args: {tool_args}
Description: {description}


Provide a comprehensive analysis covering:

1. OBVIOUS use cases (direct functionality)
2. INDIRECT connections (when this tool might be relevant)
3. TRIGGER PHRASES (what user queries would benefit from this tool)
4. COMPLEX scenarios (non-obvious applications)
5. CONTEXTUAL usage (when combined with other information)

Example for a "get_user_name" tool:
- Obvious: When user asks "what is my name"
- Indirect: Personalization, greetings, user identification
- Triggers: "my name", "who am I", "hello", "introduce yourself", "personalize"
- Complex: User context in multi-step tasks, addressing user directly
- Contextual: Any response that could be personalized

Rule! no additional comments or text in the format !
schema:
 {yaml.dump(ToolAnalysis.model_json_schema())}

Respond in YAML format:
Example:
```yaml
primary_function: "Retrieves the current user's name."
use_cases:
  - "Responding to 'what is my name?'"
  - "Personalizing greeting messages."
trigger_phrases:
  - "my name"
  - "who am I"
  - "introduce yourself"
indirect_connections:
  - "User identification in multi-factor authentication."
  - "Tagging user-generated content."
complexity_scenarios:
  - "In a multi-step task, remembering the user's name to personalize the final output."
user_intent_categories:
  - "Personalization"
  - "User Identification"
confidence_triggers:
  "my name": 0.95
  "who am I": 0.9
tool_complexity: low/medium/high
```
"""
        model = os.getenv("BASEMODEL", self.amd.fast_llm_model)
        for i in range(3):
            try:
                response = await self.a_run_llm_completion(
                    model=model,
                    messages=[{"role": "user", "content": prompt}],
                    with_context=False,
                    temperature=0.3,
                    max_tokens=1000,
                    task_id=f"tool_analysis_{tool_name}"
                )

                content = response.strip()

                # Extract JSON
                if "```yaml" in content:
                    yaml_str = content.split("```yaml")[1].split("```")[0].strip()
                else:
                    yaml_str = content

                analysis = yaml.safe_load(yaml_str)

                # Store analysis
                self._tool_capabilities[tool_name] = analysis

                # Save to cache
                await self._save_tool_analysis()

                validated_analysis = ToolAnalysis.model_validate(analysis)
                rprint(f"Generated intelligent analysis for {tool_name}")
                break

            except Exception as e:
                import traceback
                print(traceback.format_exc())
                model = self.amd.complex_llm_model if i > 1 else self.amd.fast_llm_model
                eprint(f"Tool analysis failed for {tool_name}: {e}")
                # Fallback
                self._tool_capabilities[tool_name] = {
                    "primary_function": description,
                    "use_cases": [description],
                    "trigger_phrases": [tool_name.lower().replace('_', ' ')],
                    "tool_complexity": "medium"
                }

    def _load_tool_analysis(self) -> dict[str, Any]:
        """Load tool analysis from cache"""
        try:
            if os.path.exists(self.tool_analysis_file):
                with open(self.tool_analysis_file) as f:
                    return json.load(f)
        except Exception as e:
            wprint(f"Could not load tool analysis: {e}")
        return {}


    async def save_context_to_file(self, session_id: str = None) -> bool:
        """Save current context to file"""
        try:
            context = await self.get_context(session_id=session_id, format_for_llm=False)

            filepath = self._get_context_path(session_id)

            with open(filepath, 'w', encoding='utf-8') as f:
                json.dump(context, f, indent=2, ensure_ascii=False, default=str)

            rprint(f"Context saved to: {filepath}")
            return True

        except Exception as e:
            eprint(f"Failed to save context: {e}")
            return False

    async def _save_tool_analysis(self):
        """Save tool analysis to cache"""
        try:
            with open(self.tool_analysis_file, 'w') as f:
                json.dump(self._tool_capabilities, f, indent=2)
        except Exception as e:
            eprint(f"Could not save tool analysis: {e}")

    def add_custom_flow(self, flow: AsyncFlow, name: str):
        """Add a custom flow for dynamic execution"""
        self.add_tool(flow.run_async, name=name, description=f"Custom flow: {flow.__class__.__name__}")
        rprint(f"Custom node added: {name}")

    def get_tool_by_name(self, tool_name: str) -> Callable | None:
        """Get tool function by name"""
        return self._tool_registry.get(tool_name, {}).get("function")

    async def arun_function(self, function_name: str, *args, **kwargs) -> Any:
        """
        Asynchronously finds a function by its string name, executes it with
        the given arguments, and returns the result.
        """
        rprint(f"Attempting to run function: {function_name} with args: {args}, kwargs: {kwargs}")
        target_function = self.get_tool_by_name(function_name)

        start_time = time.perf_counter()
        if not target_function:
            raise ValueError(f"Function '{function_name}' not found in the {self.amd.name}'s registered tools.")

        try:
            if asyncio.iscoroutinefunction(target_function):
                result = await target_function(*args, **kwargs)
            else:
                # If the function is not async, run it in a thread pool
                loop = asyncio.get_running_loop()
                result = await loop.run_in_executor(None, lambda: target_function(*args, **kwargs))

            if asyncio.iscoroutine(result):
                result = await result

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",  # Vereinheitlicht zu tool_call
                    node_name="FlowAgent",
                    status=NodeStatus.COMPLETED,
                    success=True,
                    duration=time.perf_counter() - start_time,
                    tool_name=function_name,
                    tool_args=kwargs,
                    tool_result=result,
                    is_meta_tool=False,  # Klarstellen, dass es kein Meta-Tool ist
                    metadata={
                        "result_type": type(result).__name__,
                        "result_length": len(str(result))
                    }
                ))
            rprint(f"Function {function_name} completed successfully with result: {result}")
            return result

        except Exception as e:
            eprint(f"Function {function_name} execution failed: {e}")
            raise

    # ===== FORMATTING =====

    async def a_format_class(self,
                             pydantic_model: type[BaseModel],
                             prompt: str,
                             message_context: list[dict] = None,
                             max_retries: int = 2, auto_context=True, session_id: str = None, **kwargs) -> dict[str, Any]:
        """
        State-of-the-art LLM-based structured data formatting using Pydantic models.

        Args:
            pydantic_model: The Pydantic model class to structure the response
            prompt: The main prompt for the LLM
            message_context: Optional conversation context messages
            max_retries: Maximum number of retry attempts

        Returns:
            dict: Validated structured data matching the Pydantic model

        Raises:
            ValidationError: If the LLM response cannot be validated against the model
            RuntimeError: If all retry attempts fail
        """

        if not LITELLM_AVAILABLE:
            raise RuntimeError("LiteLLM is required for structured formatting but not available")

        if session_id and self.active_session != session_id:
            self.active_session = session_id
        # Generate schema documentation
        schema = pydantic_model.model_json_schema() if issubclass(pydantic_model, BaseModel) else (json.loads(pydantic_model) if isinstance(pydantic_model, str) else pydantic_model)
        model_name = pydantic_model.__name__ if hasattr(pydantic_model, "__name__") else (pydantic_model.get("title", "UnknownModel") if isinstance(pydantic_model, dict) else "UnknownModel")

        # Create enhanced prompt with schema
        enhanced_prompt = f"""
    {prompt}

    CRITICAL FORMATTING REQUIREMENTS:
    1. Respond ONLY in valid YAML format
    2. Follow the exact schema structure provided
    3. Use appropriate data types (strings, lists, numbers, booleans)
    4. Include ALL required fields
    5. No additional comments, explanations, or text outside the YAML

    SCHEMA FOR {model_name}:
    {yaml.dump(schema, default_flow_style=False, indent=2)}

    EXAMPLE OUTPUT FORMAT:
    ```yaml
    # Your response here following the schema exactly
    field_name: "value"
    list_field:
      - "item1"
      - "item2"
    boolean_field: true
    number_field: 42
Respond in YAML format only:
"""
        # Prepare messages
        messages = []
        if message_context:
            messages.extend(message_context)
        messages.append({"role": "user", "content": enhanced_prompt})

        # Retry logic with progressive adjustments
        last_error = None

        for attempt in range(max_retries + 1):
            try:
                # Adjust parameters based on attempt
                temperature = 0.1 + (attempt * 0.1)  # Increase temperature slightly on retries
                max_tokens = min(2000 + (attempt * 500), 4000)  # Increase token limit on retries

                rprint(f"[{model_name}] Attempt {attempt + 1}/{max_retries + 1} (temp: {temperature})")

                # Generate LLM response
                response = await self.a_run_llm_completion(
                    model=self.amd.complex_llm_model,
                    messages=messages,
                    stream=False,
                    with_context=auto_context,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    task_id=f"format_{model_name.lower()}_{attempt}"
                )

                if not response or not response.strip():
                    raise ValueError("Empty response from LLM")

                # Extract YAML content with multiple fallback strategies

                yaml_content = self._extract_yaml_content(response)


                if not yaml_content:
                    raise ValueError("No valid YAML content found in response")

                # Parse YAML
                try:
                    parsed_data = yaml.safe_load(yaml_content)
                except yaml.YAMLError as e:
                    raise ValueError(f"Invalid YAML syntax: {e}")
                iprint(parsed_data)
                if not isinstance(parsed_data, dict):
                    raise ValueError(f"Expected dict, got {type(parsed_data)}")

                # Validate against Pydantic model
                try:
                    if isinstance(pydantic_model, BaseModel):
                        validated_instance = pydantic_model.model_validate(parsed_data)
                        validated_data = validated_instance.model_dump()
                    else:
                        validated_data = parsed_data

                    rprint(f"✅ Successfully formatted {model_name} on attempt {attempt + 1}")
                    return validated_data

                except ValidationError as e:
                    detailed_errors = []
                    for error in e.errors():
                        field_path = " -> ".join(str(x) for x in error['loc'])
                        detailed_errors.append(f"Field '{field_path}': {error['msg']}")

                    error_msg = "Validation failed:\n" + "\n".join(detailed_errors)
                    raise ValueError(error_msg)

            except Exception as e:
                last_error = e
                wprint(f"[{model_name}] Attempt {attempt + 1} failed: {str(e)}")

                if attempt < max_retries:
                    # Add error feedback for next attempt
                    error_feedback = f"\n\nPREVIOUS ATTEMPT FAILED: {str(e)}\nPlease correct the issues and provide valid YAML matching the schema exactly."
                    messages[-1]["content"] = enhanced_prompt + error_feedback

                    # Brief delay before retry
                    # await asyncio.sleep(0.5 * (attempt + 1))
                else:
                    eprint(f"[{model_name}] All {max_retries + 1} attempts failed")

        # All attempts failed
        raise RuntimeError(f"Failed to format {model_name} after {max_retries + 1} attempts. Last error: {last_error}")

    def _extract_yaml_content(self, response: str) -> str:
        """Extract YAML content from LLM response with multiple strategies"""
        # Strategy 1: Extract from code blocks
        if "```yaml" in response:
            try:
                yaml_content = response.split("```yaml")[1].split("```")[0].strip()
                if yaml_content:
                    return yaml_content
            except IndexError:
                pass

        # Strategy 2: Extract from generic code blocks
        if "```" in response:
            try:
                parts = response.split("```")
                for i, part in enumerate(parts):
                    if i % 2 == 1:  # Odd indices are inside code blocks
                        # Skip if it starts with a language identifier
                        lines = part.strip().split('\n')
                        if lines and not lines[0].strip().isalpha():
                            return part.strip()
                        elif len(lines) > 1:
                            # Try without first line
                            return '\n'.join(lines[1:]).strip()
            except:
                pass

        # Strategy 3: Look for YAML-like patterns
        lines = response.split('\n')
        yaml_lines = []
        in_yaml = False

        for line in lines:
            stripped = line.strip()

            # Detect start of YAML-like content
            if ':' in stripped and not stripped.startswith('#'):
                in_yaml = True
                yaml_lines.append(line)
            elif in_yaml:
                if stripped == '' or stripped.startswith(' ') or stripped.startswith('-') or ':' in stripped:
                    yaml_lines.append(line)
                else:
                    # Potential end of YAML
                    break

        if yaml_lines:
            return '\n'.join(yaml_lines).strip()

        # Strategy 4: Return entire response if it looks like YAML
        if ':' in response and not response.strip().startswith('<'):
            return response.strip()

        return ""
    # ===== SERVER SETUP =====

    def setup_a2a_server(self, host: str = "0.0.0.0", port: int = 5000, **kwargs):
        """Setup A2A server for bidirectional communication"""
        if not A2A_AVAILABLE:
            wprint("A2A not available, cannot setup server")
            return

        try:
            self.a2a_server = A2AServer(
                host=host,
                port=port,
                agent_card=AgentCard(
                    name=self.amd.name,
                    description="Production-ready PocketFlow agent",
                    version="1.0.0"
                ),
                **kwargs
            )

            # Register agent methods
            @self.a2a_server.route("/run")
            async def handle_run(request_data):
                query = request_data.get("query", "")
                session_id = request_data.get("session_id", "a2a_session")

                response = await self.a_run(query, session_id=session_id)
                return {"response": response}

            rprint(f"A2A server setup on {host}:{port}")

        except Exception as e:
            eprint(f"Failed to setup A2A server: {e}")

    def setup_mcp_server(self, host: str = "0.0.0.0", port: int = 8000, name: str = None, **kwargs):
        """Setup MCP server"""
        if not MCP_AVAILABLE:
            wprint("MCP not available, cannot setup server")
            return

        try:
            server_name = name or f"{self.amd.name}_MCP"
            self.mcp_server = FastMCP(server_name)

            # Register agent as MCP tool
            @self.mcp_server.tool()
            async def agent_run(query: str, session_id: str = "mcp_session") -> str:
                """Execute agent with given query"""
                return await self.a_run(query, session_id=session_id)

            rprint(f"MCP server setup: {server_name}")

        except Exception as e:
            eprint(f"Failed to setup MCP server: {e}")

    # ===== LIFECYCLE MANAGEMENT =====

    async def start_servers(self):
        """Start all configured servers"""
        tasks = []

        if self.a2a_server:
            tasks.append(asyncio.create_task(self.a2a_server.start()))

        if self.mcp_server:
            tasks.append(asyncio.create_task(self.mcp_server.run()))

        if tasks:
            rprint(f"Starting {len(tasks)} servers...")
            await asyncio.gather(*tasks, return_exceptions=True)

    def clear_context(self, session_id: str = None) -> bool:
        """Clear context über UnifiedContextManager mit Session-spezifischer Unterstützung"""
        try:
            #Clear über Context Manager
            if session_id:
                # Clear specific session
                if session_id in self.context_manager.session_managers:
                    session = self.context_manager.session_managers[session_id]
                    if hasattr(session, 'history'):
                        session.history = []
                    elif isinstance(session, dict) and 'history' in session:
                        session['history'] = []

                    # Remove from session managers
                    del self.context_manager.session_managers[session_id]

                    # Clear variable manager scope for this session
                    if self.variable_manager:
                        scope_name = f'session_{session_id}'
                        if scope_name in self.variable_manager.scopes:
                            del self.variable_manager.scopes[scope_name]

                    rprint(f"Context cleared for session: {session_id}")
            else:
                # Clear all sessions
                for session_id, session in self.context_manager.session_managers.items():
                    if hasattr(session, 'history'):
                        session.history = []
                    elif isinstance(session, dict) and 'history' in session:
                        session['history'] = []

                self.context_manager.session_managers = {}
                rprint("Context cleared for all sessions")

            # Clear context cache
            self.context_manager._invalidate_cache(session_id)

            # Clear current execution context in shared
            context_keys_to_clear = [
                "current_query", "current_response", "current_plan", "tasks",
                "results", "task_plans", "session_data", "formatted_context",
                "synthesized_response", "quality_assessment", "plan_adaptations",
                "executor_performance", "llm_tool_conversation", "aggregated_context"
            ]

            for key in context_keys_to_clear:
                if key in self.shared:
                    if isinstance(self.shared[key], dict):
                        self.shared[key] = {}
                    elif isinstance(self.shared[key], list):
                        self.shared[key] = []
                    else:
                        self.shared[key] = None

            # Clear variable manager scopes (except core system variables)
            if hasattr(self, 'variable_manager'):
                # Clear user, results, tasks scopes
                self.variable_manager.register_scope('user', {})
                self.variable_manager.register_scope('results', {})
                self.variable_manager.register_scope('tasks', {})
                # Reset cache
                self.variable_manager._cache.clear()

            # Reset execution state
            self.is_running = False
            self.is_paused = False
            self.shared["system_status"] = "idle"

            # Clear progress tracking
            if hasattr(self, 'progress_tracker'):
                self.progress_tracker.reset_session_metrics()

            return True

        except Exception as e:
            eprint(f"Failed to clear context: {e}")
            return False

    async def clean_memory(self, deep_clean: bool = False) -> bool:
        """Clean memory and context of the agent"""
        try:
            # Clear current context first
            self.clear_context()

            # Clean world model
            self.shared["world_model"] = {}
            self.world_model = {}

            # Clean performance metrics
            self.shared["performance_metrics"] = {}

            # Deep clean session storage
            session_managers = self.shared.get("session_managers", {})
            if session_managers:
                for _manager_name, manager in session_managers.items():
                    if hasattr(manager, 'clear_all_history'):
                        await manager.clear_all_history()
                    elif hasattr(manager, 'clear_history'):
                        manager.clear_history()

            # Clear session managers entirely
            self.shared["session_managers"] = {}
            self.shared["session_initialized"] = False

            # Clean variable manager completely
            if hasattr(self, 'variable_manager'):
                # Reinitialize with clean state
                self.variable_manager = VariableManager({}, self.shared)
                self._setup_variable_scopes()

            # Clean tool analysis cache if deep clean
            if deep_clean:
                self._tool_capabilities = {}
                self._tool_analysis_cache = {}

                # Remove tool analysis file
                if hasattr(self, 'tool_analysis_file') and os.path.exists(self.tool_analysis_file):
                    try:
                        os.remove(self.tool_analysis_file)
                        rprint("Removed tool analysis cache file")
                    except:
                        pass

            # Clean checkpoint data
            self.checkpoint_data = {}
            self.last_checkpoint = None

            # Clean execution history
            if hasattr(self.task_flow, 'executor_node'):
                self.task_flow.executor_node.execution_history = []
                self.task_flow.executor_node.results_store = {}

            # Clean context manager sessions
            if hasattr(self.task_flow, 'context_manager'):
                self.task_flow.context_manager.session_managers = {}

            # Clean LLM call statistics
            self.shared.pop("llm_call_stats", None)

            # Force garbage collection
            import gc
            gc.collect()

            rprint(f"Memory cleaned (deep_clean: {deep_clean})")
            return True

        except Exception as e:
            eprint(f"Failed to clean memory: {e}")
            return False

    async def close(self):
        """Clean shutdown"""
        self.is_running = False
        self._shutdown_event.set()

        # Create final checkpoint
        if self.enable_pause_resume:
            checkpoint = await self._create_checkpoint()
            await self._save_checkpoint(checkpoint, "final_checkpoint.pkl")

        # Shutdown executor
        self.executor.shutdown(wait=True)

        # Close servers
        if self.a2a_server:
            await self.a2a_server.close()

        if self.mcp_server:
            await self.mcp_server.close()

        if hasattr(self, '_mcp_session_manager'):
            await self._mcp_session_manager.cleanup_all()

        rprint("Agent shutdown complete")

    @property
    def total_cost(self) -> float:
        """Get total cost if budget manager available"""
        if hasattr(self.amd, 'budget_manager') and self.amd.budget_manager:
            return getattr(self.amd.budget_manager, 'total_cost', 0.0)
        return 0.0

    async def get_context_overview(self, session_id: str = None, display: bool = False) -> dict[str, Any]:
        """
        Detaillierte Übersicht des aktuellen Contexts mit Token-Counts und optionaler Display-Darstellung

        Args:
            session_id: Session ID für context (default: active_session)
            display: Ob die Übersicht im Terminal-Style angezeigt werden soll

        Returns:
            dict: Detaillierte Context-Übersicht mit Raw-Daten und Token-Counts
        """
        try:
            session_id = session_id or self.active_session or "default"

            # Token counting function
            def count_tokens(text: str) -> int:
                """Einfache Token-Approximation (4 chars ≈ 1 token für deutsche/englische Texte)"""
                try:
                    from litellm import token_counter
                    return token_counter(self.amd.fast_llm_model, text=text)
                except:
                    pass
                return max(1, len(str(text)) // 4)

            context_overview = {
                "session_info": {
                    "session_id": session_id,
                    "agent_name": self.amd.name,
                    "timestamp": datetime.now().isoformat(),
                    "active_session": self.active_session,
                    "is_running": self.is_running
                },
                "system_prompt": {},
                "meta_tools": {},
                "agent_tools": {},
                "mcp_tools": {},
                "variables": {},
                "system_history": {},
                "unified_context": {},
                "reasoning_context": {},
                "llm_tool_context": {},
                "token_summary": {}
            }

            # === SYSTEM PROMPT ANALYSIS ===
            system_message = self.amd.get_system_message_with_persona()
            context_overview["system_prompt"] = {
                "raw_data": system_message,
                "token_count": count_tokens(system_message),
                "components": {
                    "base_message": self.amd.system_message,
                    "persona_active": self.amd.persona is not None,
                    "persona_name": self.amd.persona.name if self.amd.persona else None,
                    "persona_integration": self.amd.persona.apply_method if self.amd.persona else None
                }
            }

            # === META TOOLS ANALYSIS ===
            if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
                meta_tools = self.task_flow.llm_reasoner.meta_tools_registry
            else:
                meta_tools = {}

            meta_tools_info = ""
            for tool_name, tool_info in meta_tools.items():
                tool_desc = tool_info.get("description", "No description")
                meta_tools_info += f"{tool_name}: {tool_desc}\n"

            # Standard Meta-Tools
            standard_meta_tools = [
                "internal_reasoning", "manage_internal_task_stack", "delegate_to_llm_tool_node",
                "create_and_execute_plan", "advance_outline_step", "write_to_variables",
                "read_from_variables", "direct_response"
            ]

            for meta_tool in standard_meta_tools:
                meta_tools_info += f"{meta_tool}: Built-in meta-tool for agent orchestration\n"

            context_overview["meta_tools"] = {
                "raw_data": meta_tools_info,
                "token_count": count_tokens(meta_tools_info),
                "count": len(meta_tools) + len(standard_meta_tools),
                "custom_meta_tools": list(meta_tools.keys()),
                "standard_meta_tools": standard_meta_tools
            }

            # === AGENT TOOLS ANALYSIS ===
            tools_info = ""
            tool_capabilities_text = ""

            for tool_name in self.shared.get("available_tools", []):
                tool_data = self._tool_registry.get(tool_name, {})
                description = tool_data.get("description", "No description")
                args_schema = tool_data.get("args_schema", "()")
                tools_info += f"{tool_name}{args_schema}: {description}\n"

                # Tool capabilities if available
                if tool_name in self._tool_capabilities:
                    cap = self._tool_capabilities[tool_name]
                    primary_function = cap.get("primary_function", "Unknown")
                    use_cases = cap.get("use_cases", [])
                    tool_capabilities_text += f"{tool_name}: {primary_function}\n"
                    if use_cases:
                        tool_capabilities_text += f"  Use cases: {', '.join(use_cases[:3])}\n"

            context_overview["agent_tools"] = {
                "raw_data": tools_info,
                "capabilities_data": tool_capabilities_text,
                "token_count": count_tokens(tools_info + tool_capabilities_text),
                "count": len(self.shared.get("available_tools", [])),
                "analyzed_count": len(self._tool_capabilities),
                "tool_names": self.shared.get("available_tools", []),
                "intelligence_level": "high" if self._tool_capabilities else "basic"
            }

            # === MCP TOOLS ANALYSIS ===
            # Placeholder für MCP Tools (falls implementiert)
            mcp_tools_info = "No MCP tools currently active"
            if self.mcp_server:
                mcp_tools_info = f"MCP Server active: {getattr(self.mcp_server, 'name', 'Unknown')}"

            context_overview["mcp_tools"] = {
                "raw_data": mcp_tools_info,
                "token_count": count_tokens(mcp_tools_info),
                "server_active": bool(self.mcp_server),
                "server_name": getattr(self.mcp_server, 'name', None) if self.mcp_server else None
            }

            # === VARIABLES ANALYSIS ===
            variables_text = ""
            if self.variable_manager:
                variables_text = self.variable_manager.get_llm_variable_context()
            else:
                variables_text = "No variable manager available"

            context_overview["variables"] = {
                "raw_data": variables_text,
                "token_count": count_tokens(variables_text),
                "manager_available": bool(self.variable_manager),
                "total_scopes": len(self.variable_manager.scopes) if self.variable_manager else 0,
                "scope_names": list(self.variable_manager.scopes.keys()) if self.variable_manager else []
            }

            # === SYSTEM HISTORY ANALYSIS ===
            history_text = ""
            if self.context_manager and session_id in self.context_manager.session_managers:
                session = self.context_manager.session_managers[session_id]
                if hasattr(session, 'history'):
                    history_count = len(session.history)
                    history_text = f"Session History: {history_count} messages\n"

                    # Recent messages preview
                    for msg in session.history[-3:]:
                        role = msg.get('role', 'unknown')
                        content = msg.get('content', '')[:100] + "..." if len(
                            msg.get('content', '')) > 100 else msg.get('content', '')
                        timestamp = msg.get('timestamp', '')[:19]
                        history_text += f"[{timestamp}] {role}: {content}\n"
                elif isinstance(session, dict) and 'history' in session:
                    history_count = len(session['history'])
                    history_text = f"Fallback Session History: {history_count} messages"
            else:
                history_text = "No session history available"

            context_overview["system_history"] = {
                "raw_data": history_text,
                "token_count": count_tokens(history_text),
                "session_initialized": self.shared.get("session_initialized", False),
                "context_manager_available": bool(self.context_manager),
                "session_count": len(self.context_manager.session_managers) if self.context_manager else 0
            }

            # === UNIFIED CONTEXT ANALYSIS ===
            unified_context_text = ""
            try:
                unified_context = await self.context_manager.build_unified_context(session_id, "",
                                                                                   "full") if self.context_manager else {}
                if unified_context:
                    formatted_context = self.context_manager.get_formatted_context_for_llm(unified_context)
                    unified_context_text = formatted_context
                else:
                    unified_context_text = "No unified context available"
            except Exception as e:
                unified_context_text = f"Error building unified context: {str(e)}"

            context_overview["unified_context"] = {
                "raw_data": unified_context_text,
                "token_count": count_tokens(unified_context_text),
                "build_successful": "Error" not in unified_context_text,
                "manager_available": bool(self.context_manager)
            }

            # === REASONING CONTEXT ANALYSIS ===
            reasoning_context_text = ""
            if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'reasoning_context'):
                reasoning_context = self.task_flow.llm_reasoner.reasoning_context
                reasoning_context_text = f"Reasoning Context: {len(reasoning_context)} entries\n"

                # Recent reasoning entries
                for entry in reasoning_context[-3:]:
                    entry_type = entry.get('type', 'unknown')
                    content = str(entry.get('content', ''))[:150] + "..." if len(
                        str(entry.get('content', ''))) > 150 else str(entry.get('content', ''))
                    reasoning_context_text += f"  {entry_type}: {content}\n"
            else:
                reasoning_context_text = "No reasoning context available"

            context_overview["reasoning_context"] = {
                "raw_data": reasoning_context_text,
                "token_count": count_tokens(reasoning_context_text),
                "reasoner_available": hasattr(self.task_flow, 'llm_reasoner'),
                "context_entries": len(self.task_flow.llm_reasoner.reasoning_context) if hasattr(self.task_flow,
                                                                                                 'llm_reasoner') and hasattr(
                    self.task_flow.llm_reasoner, 'reasoning_context') else 0
            }

            # === LLM TOOL CONTEXT ANALYSIS ===
            llm_tool_context_text = ""
            if hasattr(self.task_flow, 'llm_tool_node'):
                llm_tool_context_text = f"LLM Tool Node available with max {self.task_flow.llm_tool_node.max_tool_calls} tool calls\n"
                if hasattr(self.task_flow.llm_tool_node, 'call_log'):
                    call_log = self.task_flow.llm_tool_node.call_log
                    llm_tool_context_text += f"Call log: {len(call_log)} entries\n"
            else:
                llm_tool_context_text = "No LLM Tool Node available"

            context_overview["llm_tool_context"] = {
                "raw_data": llm_tool_context_text,
                "token_count": count_tokens(llm_tool_context_text),
                "node_available": hasattr(self.task_flow, 'llm_tool_node'),
                "max_tool_calls": getattr(self.task_flow.llm_tool_node, 'max_tool_calls', 0) if hasattr(self.task_flow,
                                                                                                        'llm_tool_node') else 0
            }

            # === TOKEN SUMMARY ===
            total_tokens = sum([
                context_overview["system_prompt"]["token_count"],
                context_overview["meta_tools"]["token_count"],
                context_overview["agent_tools"]["token_count"],
                context_overview["mcp_tools"]["token_count"],
                context_overview["variables"]["token_count"],
                context_overview["system_history"]["token_count"],
                context_overview["unified_context"]["token_count"],
                context_overview["reasoning_context"]["token_count"],
                context_overview["llm_tool_context"]["token_count"]
            ])

            context_overview["token_summary"] = {
                "total_tokens": total_tokens,
                "breakdown": {
                    "system_prompt": context_overview["system_prompt"]["token_count"],
                    "meta_tools": context_overview["meta_tools"]["token_count"],
                    "agent_tools": context_overview["agent_tools"]["token_count"],
                    "mcp_tools": context_overview["mcp_tools"]["token_count"],
                    "variables": context_overview["variables"]["token_count"],
                    "system_history": context_overview["system_history"]["token_count"],
                    "unified_context": context_overview["unified_context"]["token_count"],
                    "reasoning_context": context_overview["reasoning_context"]["token_count"],
                    "llm_tool_context": context_overview["llm_tool_context"]["token_count"]
                },
                "percentage_breakdown": {}
            }

            # Calculate percentages
            for component, token_count in context_overview["token_summary"]["breakdown"].items():
                percentage = (token_count / total_tokens * 100) if total_tokens > 0 else 0
                context_overview["token_summary"]["percentage_breakdown"][component] = round(percentage, 1)

            # === DISPLAY OUTPUT ===
            if display:
                await self._display_context_overview(context_overview)

            return context_overview

        except Exception as e:
            eprint(f"Error generating context overview: {e}")
            return {
                "error": str(e),
                "timestamp": datetime.now().isoformat(),
                "session_id": session_id
            }

    async def _display_context_overview(self, overview: dict[str, Any]):
        """Display context overview in terminal-style format similar to the image"""
        try:
            from toolboxv2.utils.extras.Style import Spinner

            print("\n" + "=" * 80)
            print("🔍 FLOW AGENT CONTEXT OVERVIEW")
            print("=" * 80)

            # Session Info
            session_info = overview["session_info"]
            print(f"📅 Session: {session_info['session_id']} | Agent: {session_info['agent_name']}")
            print(f"⏰ Generated: {session_info['timestamp'][:19]} | Running: {session_info['is_running']}")

            # Token Summary (like in the image)
            token_summary = overview["token_summary"]
            total_tokens = token_summary["total_tokens"]
            breakdown = token_summary["percentage_breakdown"]

            print(f"\n📊 CONTEXT USAGE")
            print(f"Total Context: ~{total_tokens:,} tokens")

            # Create visual bars like in the image
            bar_length = 50

            try:mf=get_max_tokens(self.amd.fast_llm_model.split('/')[-1]);self.amd.max_tokens = mf
            except:mf = self.amd.max_tokens
            try:mc=get_max_tokens(self.amd.complex_llm_model.split('/')[-1]);self.amd.max_tokens = mf
            except:mc = self.amd.max_tokens
            components = [
                ("System prompt", breakdown.get("system_prompt", 0), "🔧"),
                ("Agent tools", breakdown.get("agent_tools", 0), "🛠️"),
                ("Meta tools", breakdown.get("meta_tools", 0), "⚡"),
                ("Variables", breakdown.get("variables", 0), "📝"),
                ("History", breakdown.get("system_history", 0), "📚"),
                ("Unified ctx", breakdown.get("unified_context", 0), "🔗"),
                ("Reasoning", breakdown.get("reasoning_context", 0), "🧠"),
                ("LLM Tools", breakdown.get("llm_tool_context", 0), "🤖"),
                ("Free Space F", mf, "⬜"),
                ("Free Space C", mc, "⬜"),

            ]

            for name, percentage, icon in components:
                if percentage > 0:
                    filled_length = int(percentage * bar_length / 100)
                    bar = "█" * filled_length + "░" * (bar_length - filled_length)
                    tokens = int(total_tokens * percentage / 100)
                    print(f"{icon} {name:13}: {bar} {percentage:5.1f}% ({tokens:,} tokens)") if not name.startswith("Free") else print(f"{icon} {name:13}: ({tokens:,} tokens) used {total_tokens/tokens*100:.3f}%")

            # Detailed breakdowns
            sections = [
                ("🔧 SYSTEM PROMPT", "system_prompt"),
                ("⚡ META TOOLS", "meta_tools"),
                ("🛠️ AGENT TOOLS", "agent_tools"),
                ("📝 VARIABLES", "variables"),
                ("📚 SYSTEM HISTORY", "system_history"),
                ("🔗 UNIFIED CONTEXT", "unified_context"),
                ("🧠 REASONING CONTEXT", "reasoning_context"),
                ("🤖 LLM TOOL CONTEXT", "llm_tool_context")
            ]

            for title, key in sections:
                section_data = overview.get(key, {})
                token_count = section_data.get("token_count", 0)

                if token_count > 0:
                    print(f"\n{title} ({token_count:,} tokens)")
                    print("-" * 50)

                    # Show component-specific info
                    if key == "agent_tools":
                        print(f"  Available tools: {section_data.get('count', 0)}")
                        print(f"  Analyzed tools: {section_data.get('analyzed_count', 0)}")
                        print(f"  Intelligence: {section_data.get('intelligence_level', 'unknown')}")
                    elif key == "variables":
                        print(f"  Manager available: {section_data.get('manager_available', False)}")
                        print(f"  Total scopes: {section_data.get('total_scopes', 0)}")
                        print(f"  Scope names: {', '.join(section_data.get('scope_names', []))}")
                    elif key == "system_history":
                        print(f"  Session initialized: {section_data.get('session_initialized', False)}")
                        print(f"  Total sessions: {section_data.get('session_count', 0)}")
                    elif key == "reasoning_context":
                        print(f"  Reasoner available: {section_data.get('reasoner_available', False)}")
                        print(f"  Context entries: {section_data.get('context_entries', 0)}")
                    elif key == "meta_tools":
                        print(f"  Total meta tools: {section_data.get('count', 0)}")
                        custom = section_data.get('custom_meta_tools', [])
                        if custom:
                            print(f"  Custom tools: {', '.join(custom)}")

                    # Show raw data preview if reasonable size
                    raw_data = section_data.get('raw_data', '')
                    if len(raw_data) <= 200:
                        print(f"  Preview: {raw_data[:200]}...")

            print("\n" + "=" * 80)
            print(f"💾 Total Context Size: ~{total_tokens:,} tokens")
            print("=" * 80 + "\n")

        except Exception as e:
            eprint(f"Error displaying context overview: {e}")
            # Fallback to simple display
            print(f"\n=== CONTEXT OVERVIEW (Fallback) ===")
            print(f"Total Tokens: {overview.get('token_summary', {}).get('total_tokens', 0):,}")
            for key, data in overview.items():
                if isinstance(data, dict) and 'token_count' in data:
                    print(f"{key}: {data['token_count']:,} tokens")
            print("=" * 40)

    async def status(self, pretty_print: bool = False) -> dict[str, Any] | str:
        """Get comprehensive agent status with optional pretty printing"""

        # Core status information
        base_status = {
            "agent_info": {
                "name": self.amd.name,
                "version": "2.0",
                "type": "FlowAgent"
            },
            "runtime_status": {
                "status": self.shared.get("system_status", "idle"),
                "is_running": self.is_running,
                "is_paused": self.is_paused,
                "uptime_seconds": (datetime.now() - getattr(self, '_start_time', datetime.now())).total_seconds()
            },
            "task_execution": {
                "total_tasks": len(self.shared.get("tasks", {})),
                "active_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "running"]),
                "completed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "completed"]),
                "failed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "failed"]),
                "plan_adaptations": self.shared.get("plan_adaptations", 0)
            },
            "conversation": {
                "turns": len(self.shared.get("conversation_history", [])),
                "session_id": self.shared.get("session_id", self.active_session),
                "current_user": self.shared.get("user_id"),
                "last_query": self.shared.get("current_query", "")[:100] + "..." if len(
                    self.shared.get("current_query", "")) > 100 else self.shared.get("current_query", "")
            },
            "capabilities": {
                "available_tools": len(self.shared.get("available_tools", [])),
                "tool_names": list(self.shared.get("available_tools", [])),
                "analyzed_tools": len(self._tool_capabilities),
                "world_model_size": len(self.shared.get("world_model", {})),
                "intelligence_level": "high" if self._tool_capabilities else "basic"
            },
            "memory_context": {
                "session_initialized": self.shared.get("session_initialized", False),
                "session_managers": len(self.shared.get("session_managers", {})),
                "context_system": "advanced_session_aware" if self.shared.get("session_initialized") else "basic",
                "variable_scopes": len(self.variable_manager.get_scope_info()) if hasattr(self,
                                                                                          'variable_manager') else 0
            },
            "performance": {
                "total_cost": self.total_cost,
                "checkpoint_enabled": self.enable_pause_resume,
                "last_checkpoint": self.last_checkpoint.isoformat() if self.last_checkpoint else None,
                "max_parallel_tasks": self.max_parallel_tasks
            },
            "servers": {
                "a2a_server": self.a2a_server is not None,
                "mcp_server": self.mcp_server is not None,
                "server_count": sum([self.a2a_server is not None, self.mcp_server is not None])
            },
            "configuration": {
                "fast_llm_model": self.amd.fast_llm_model,
                "complex_llm_model": self.amd.complex_llm_model,
                "use_fast_response": getattr(self.amd, 'use_fast_response', False),
                "max_input_tokens": getattr(self.amd, 'max_input_tokens', 8000),
                "persona_configured": self.amd.persona is not None,
                "format_config": bool(getattr(self.amd.persona, 'format_config', None)) if self.amd.persona else False
            }
        }

        # Add detailed execution summary if tasks exist
        tasks = self.shared.get("tasks", {})
        if tasks:
            task_types_used = {}
            tools_used = []
            execution_timeline = []

            for task_id, task in tasks.items():
                # Count task types
                task_type = getattr(task, 'type', 'unknown')
                task_types_used[task_type] = task_types_used.get(task_type, 0) + 1

                # Collect tools used
                if hasattr(task, 'tool_name') and task.tool_name:
                    tools_used.append(task.tool_name)

                # Timeline info
                if hasattr(task, 'started_at') and task.started_at:
                    timeline_entry = {
                        "task_id": task_id,
                        "type": task_type,
                        "started": task.started_at.isoformat(),
                        "status": getattr(task, 'status', 'unknown')
                    }
                    if hasattr(task, 'completed_at') and task.completed_at:
                        timeline_entry["completed"] = task.completed_at.isoformat()
                        timeline_entry["duration"] = (task.completed_at - task.started_at).total_seconds()
                    execution_timeline.append(timeline_entry)

            base_status["task_execution"].update({
                "task_types_used": task_types_used,
                "tools_used": list(set(tools_used)),
                "execution_timeline": execution_timeline[-5:]  # Last 5 tasks
            })

        # Add context statistics
        if hasattr(self.task_flow, 'context_manager'):
            context_manager = self.task_flow.context_manager
            base_status["memory_context"].update({
                "compression_threshold": context_manager.compression_threshold,
                "max_tokens": context_manager.max_tokens,
                "active_context_sessions": len(getattr(context_manager, 'session_managers', {}))
            })

        # Add variable system info
        if hasattr(self, 'variable_manager'):
            available_vars = self.variable_manager.get_available_variables()
            scope_info = self.variable_manager.get_scope_info()

            base_status["variable_system"] = {
                "total_scopes": len(scope_info),
                "scope_names": list(scope_info.keys()),
                "total_variables": sum(len(vars) for vars in available_vars.values()),
                "scope_details": {
                    scope: {"type": info["type"], "variables": len(available_vars.get(scope, {}))}
                    for scope, info in scope_info.items()
                }
            }

        # Add format quality info if available
        quality_assessment = self.shared.get("quality_assessment", {})
        if quality_assessment:
            quality_details = quality_assessment.get("quality_details", {})
            base_status["format_quality"] = {
                "overall_score": quality_details.get("total_score", 0.0),
                "format_adherence": quality_details.get("format_adherence", 0.0),
                "length_adherence": quality_details.get("length_adherence", 0.0),
                "content_quality": quality_details.get("base_quality", 0.0),
                "assessment": quality_assessment.get("quality_assessment", "unknown"),
                "has_suggestions": bool(quality_assessment.get("suggestions", []))
            }

        # Add LLM usage statistics
        llm_stats = self.shared.get("llm_call_stats", {})
        if llm_stats:
            base_status["llm_usage"] = {
                "total_calls": llm_stats.get("total_calls", 0),
                "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
                "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                        1),
                "total_tokens_used": llm_stats.get("total_tokens_used", 0)
            }

        # Add timestamp
        base_status["timestamp"] = datetime.now().isoformat()

        base_status["context_statistic"] = self.get_context_statistics()
        if not pretty_print:
            base_status["agent_context"] = await self.get_context_overview()
            return base_status

        # Pretty print using EnhancedVerboseOutput
        try:
            from toolboxv2.mods.isaa.extras.verbose_output import EnhancedVerboseOutput
            verbose_output = EnhancedVerboseOutput(verbose=True)

            # Header
            verbose_output.log_header(f"Agent Status: {base_status['agent_info']['name']}")

            # Runtime Status
            status_color = {
                "running": "SUCCESS",
                "paused": "WARNING",
                "idle": "INFO",
                "error": "ERROR"
            }.get(base_status["runtime_status"]["status"], "INFO")

            getattr(verbose_output, f"print_{status_color.lower()}")(
                f"Status: {base_status['runtime_status']['status'].upper()}"
            )

            # Task Execution Summary
            task_exec = base_status["task_execution"]
            if task_exec["total_tasks"] > 0:
                verbose_output.formatter.print_section(
                    "Task Execution",
                    f"Total: {task_exec['total_tasks']} | "
                    f"Completed: {task_exec['completed_tasks']} | "
                    f"Failed: {task_exec['failed_tasks']} | "
                    f"Active: {task_exec['active_tasks']}\n"
                    f"Adaptations: {task_exec['plan_adaptations']}"
                )

                if task_exec.get("tools_used"):
                    verbose_output.formatter.print_section(
                        "Tools Used",
                        ", ".join(task_exec["tools_used"])
                    )

            # Capabilities
            caps = base_status["capabilities"]
            verbose_output.formatter.print_section(
                "Capabilities",
                f"Intelligence Level: {caps['intelligence_level']}\n"
                f"Available Tools: {caps['available_tools']}\n"
                f"Analyzed Tools: {caps['analyzed_tools']}\n"
                f"World Model Size: {caps['world_model_size']}"
            )

            # Memory & Context
            memory = base_status["memory_context"]
            verbose_output.formatter.print_section(
                "Memory & Context",
                f"Context System: {memory['context_system']}\n"
                f"Session Managers: {memory['session_managers']}\n"
                f"Variable Scopes: {memory['variable_scopes']}\n"
                f"Session Initialized: {memory['session_initialized']}"
            )

            # Context Statistics
            stats = base_status["context_statistic"]
            verbose_output.formatter.print_section(
                "Context & Stats",
                f"Compression Stats: {stats['compression_stats']}\n"
                f"Session Usage: {stats['context_usage']}\n"
                f"Session Managers: {stats['session_managers']}\n"
            )

            # Configuration
            config = base_status["configuration"]
            verbose_output.formatter.print_section(
                "Configuration",
                f"Fast LLM: {config['fast_llm_model']}\n"
                f"Complex LLM: {config['complex_llm_model']}\n"
                f"Max Tokens: {config['max_input_tokens']}\n"
                f"Persona: {'Configured' if config['persona_configured'] else 'Default'}\n"
                f"Format Config: {'Active' if config['format_config'] else 'None'}"
            )

            # Performance
            perf = base_status["performance"]
            verbose_output.formatter.print_section(
                "Performance",
                f"Total Cost: ${perf['total_cost']:.4f}\n"
                f"Checkpointing: {'Enabled' if perf['checkpoint_enabled'] else 'Disabled'}\n"
                f"Max Parallel Tasks: {perf['max_parallel_tasks']}\n"
                f"Last Checkpoint: {perf['last_checkpoint'] or 'None'}"
            )

            # Variable System Details
            if "variable_system" in base_status:
                var_sys = base_status["variable_system"]
                scope_details = []
                for scope, details in var_sys["scope_details"].items():
                    scope_details.append(f"{scope}: {details['variables']} variables ({details['type']})")

                verbose_output.formatter.print_section(
                    "Variable System",
                    f"Total Scopes: {var_sys['total_scopes']}\n"
                    f"Total Variables: {var_sys['total_variables']}\n" +
                    "\n".join(scope_details)
                )

            # Format Quality
            if "format_quality" in base_status:
                quality = base_status["format_quality"]
                verbose_output.formatter.print_section(
                    "Format Quality",
                    f"Overall Score: {quality['overall_score']:.2f}\n"
                    f"Format Adherence: {quality['format_adherence']:.2f}\n"
                    f"Length Adherence: {quality['length_adherence']:.2f}\n"
                    f"Content Quality: {quality['content_quality']:.2f}\n"
                    f"Assessment: {quality['assessment']}"
                )

            # LLM Usage
            if "llm_usage" in base_status:
                llm = base_status["llm_usage"]
                verbose_output.formatter.print_section(
                    "LLM Usage Statistics",
                    f"Total Calls: {llm['total_calls']}\n"
                    f"Avg Context Tokens: {llm['average_context_tokens']:.1f}\n"
                    f"Total Tokens: {llm['total_tokens_used']}\n"
                    f"Compression Rate: {llm['context_compression_rate']:.2%}"
                )

            # Servers
            servers = base_status["servers"]
            if servers["server_count"] > 0:
                server_status = []
                if servers["a2a_server"]:
                    server_status.append("A2A Server: Active")
                if servers["mcp_server"]:
                    server_status.append("MCP Server: Active")

                verbose_output.formatter.print_section(
                    "Servers",
                    "\n".join(server_status)
                )

            verbose_output.print_separator()
            await self.get_context_overview(display=True)
            verbose_output.print_separator()
            verbose_output.print_info(f"Status generated at: {base_status['timestamp']}")

            return "Status printed above"

        except Exception:
            # Fallback to JSON if pretty print fails
            import json
            return json.dumps(base_status, indent=2, default=str)

    @property
    def tool_registry(self):
        return self._tool_registry

    def __rshift__(self, other):
        return Chain(self) >> other

    def __add__(self, other):
        return Chain(self) + other

    def __and__(self, other):
        return Chain(self) & other

    def __mod__(self, other):
        """Implements % operator for conditional branching"""
        return ConditionalChain(self, other)

    def bind(self, *agents, shared_scopes: list[str] = None, auto_sync: bool = True):
        """
        Bind two or more agents together with shared and private variable spaces.

        Args:
            *agents: FlowAgent instances to bind together
            shared_scopes: List of scope names to share (default: ['world', 'results', 'system'])
            auto_sync: Whether to automatically sync variables and context

        Returns:
            dict: Binding configuration with agent references
        """
        if shared_scopes is None:
            shared_scopes = ['world', 'results', 'system']

        # Create unique binding ID
        binding_id = f"bind_{int(time.time())}_{len(agents)}"

        # All agents in this binding (including self)
        all_agents = [self] + list(agents)

        # Create shared variable manager that all agents will reference
        shared_world_model = {}
        shared_state = {}

        # Merge existing data from all agents
        for agent in all_agents:
            # Merge world models
            shared_world_model.update(agent.world_model)
            shared_state.update(agent.shared)

        # Create shared variable manager
        shared_variable_manager = VariableManager(shared_world_model, shared_state)

        # Set up shared scopes with merged data
        for scope_name in shared_scopes:
            merged_scope = {}
            for agent in all_agents:
                if hasattr(agent, 'variable_manager') and agent.variable_manager:
                    agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                    if isinstance(agent_scope_data, dict):
                        merged_scope.update(agent_scope_data)
            shared_variable_manager.register_scope(scope_name, merged_scope)

        # Create binding configuration
        binding_config = {
            'binding_id': binding_id,
            'agents': all_agents,
            'shared_scopes': shared_scopes,
            'auto_sync': auto_sync,
            'shared_variable_manager': shared_variable_manager,
            'private_managers': {},
            'created_at': datetime.now().isoformat()
        }

        # Configure each agent
        for i, agent in enumerate(all_agents):
            agent_private_id = f"{binding_id}_agent_{i}_{agent.amd.name}"

            # Create private variable manager for agent-specific data
            private_world_model = agent.world_model.copy()
            private_shared = agent.shared.copy()
            private_variable_manager = VariableManager(private_world_model, private_shared)

            # Set up private scopes (user, session-specific data, agent-specific configs)
            private_scopes = ['user', 'agent', 'session_private', 'tasks_private']
            for scope_name in private_scopes:
                if hasattr(agent, 'variable_manager') and agent.variable_manager:
                    agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                    private_variable_manager.register_scope(f"{scope_name}_{agent.amd.name}", agent_scope_data)

            binding_config['private_managers'][agent.amd.name] = private_variable_manager

            # Replace agent's variable manager with a unified one
            unified_manager = UnifiedBindingManager(
                shared_manager=shared_variable_manager,
                private_manager=private_variable_manager,
                agent_name=agent.amd.name,
                shared_scopes=shared_scopes,
                auto_sync=auto_sync,
                binding_config=binding_config
            )

            # Store original managers for unbinding
            if not hasattr(agent, '_original_managers'):
                agent._original_managers = {
                    'variable_manager': agent.variable_manager,
                    'world_model': agent.world_model.copy(),
                    'shared': agent.shared.copy()
                }

            # Set new unified manager
            agent.variable_manager = unified_manager
            agent.world_model = shared_world_model
            agent.shared = shared_state

            # Update shared state with binding info
            agent.shared['binding_config'] = binding_config
            agent.shared['is_bound'] = True
            agent.shared['binding_id'] = binding_id
            agent.shared['bound_agents'] = [a.amd.name for a in all_agents]

            # Sync context manager if available
            if hasattr(agent, 'context_manager') and agent.context_manager:
                agent.context_manager.variable_manager = unified_manager

                # Share session managers between bound agents if auto_sync is enabled
                if auto_sync:
                    # Merge session managers from all agents
                    all_sessions = {}
                    for bound_agent in all_agents:
                        if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                            if hasattr(bound_agent.context_manager, 'session_managers'):
                                all_sessions.update(bound_agent.context_manager.session_managers)

                    # Update all agents with merged sessions
                    for bound_agent in all_agents:
                        if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                            bound_agent.context_manager.session_managers.update(all_sessions)

        # Set up auto-sync mechanism if enabled
        if auto_sync:
            binding_config['sync_handler'] = BindingSyncHandler(binding_config)

        rprint(f"Successfully bound {len(all_agents)} agents together (Binding ID: {binding_id})")
        rprint(f"Shared scopes: {', '.join(shared_scopes)}")
        rprint(f"Bound agents: {', '.join([agent.amd.name for agent in all_agents])}")

        return binding_config

    def unbind(self, preserve_shared_data: bool = False):
        """
        Unbind this agent from any binding configuration.

        Args:
            preserve_shared_data: Whether to preserve shared data in the agent after unbinding

        Returns:
            dict: Unbinding result with statistics
        """
        if not self.shared.get('is_bound', False):
            return {
                'success': False,
                'message': f"Agent {self.amd.name} is not currently bound to any other agents"
            }

        binding_config = self.shared.get('binding_config')
        if not binding_config:
            return {
                'success': False,
                'message': "No binding configuration found"
            }

        binding_id = binding_config['binding_id']
        bound_agents = binding_config['agents']

        unbind_stats = {
            'binding_id': binding_id,
            'agents_affected': [],
            'shared_data_preserved': preserve_shared_data,
            'private_data_restored': False,
            'unbind_timestamp': datetime.now().isoformat()
        }

        try:
            # Restore original managers for this agent
            if hasattr(self, '_original_managers'):
                original = self._original_managers

                if preserve_shared_data:
                    # Merge current shared data with original data
                    if isinstance(original['world_model'], dict):
                        original['world_model'].update(self.world_model)
                    if isinstance(original['shared'], dict):
                        original['shared'].update({k: v for k, v in self.shared.items()
                                                   if k not in ['binding_config', 'is_bound', 'binding_id',
                                                                'bound_agents']})

                # Restore original variable manager
                self.variable_manager = original['variable_manager']
                self.world_model = original['world_model']
                self.shared = original['shared']

                # Update context manager
                if hasattr(self, 'context_manager') and self.context_manager:
                    self.context_manager.variable_manager = self.variable_manager

                unbind_stats['private_data_restored'] = True
                del self._original_managers

            # Clean up binding state
            self.shared.pop('binding_config', None)
            self.shared.pop('is_bound', None)
            self.shared.pop('binding_id', None)
            self.shared.pop('bound_agents', None)

            # Update binding configuration to remove this agent
            remaining_agents = [agent for agent in bound_agents if agent != self]
            if remaining_agents:
                # Update binding config for remaining agents
                binding_config['agents'] = remaining_agents
                for agent in remaining_agents:
                    if hasattr(agent, 'shared') and agent.shared.get('is_bound'):
                        agent.shared['bound_agents'] = [a.amd.name for a in remaining_agents]

            unbind_stats['agents_affected'] = [agent.amd.name for agent in bound_agents]

            # Clean up sync handler if this was the last agent
            if len(remaining_agents) <= 1:
                sync_handler = binding_config.get('sync_handler')
                if sync_handler and hasattr(sync_handler, 'cleanup'):
                    sync_handler.cleanup()

            rprint(f"Agent {self.amd.name} successfully unbound from binding {binding_id}")
            rprint(f"Shared data preserved: {preserve_shared_data}")

            return {
                'success': True,
                'stats': unbind_stats,
                'message': f"Agent {self.amd.name} unbound successfully"
            }

        except Exception as e:
            eprint(f"Error during unbinding: {e}")
            return {
                'success': False,
                'error': str(e),
                'stats': unbind_stats
            }
total_cost property

Get total cost if budget manager available

__mod__(other)

Implements % operator for conditional branching

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11458
11459
11460
def __mod__(self, other):
    """Implements % operator for conditional branching"""
    return ConditionalChain(self, other)
a_format_class(pydantic_model, prompt, message_context=None, max_retries=2, auto_context=True, session_id=None, **kwargs) async

State-of-the-art LLM-based structured data formatting using Pydantic models.

Parameters:

Name Type Description Default
pydantic_model type[BaseModel]

The Pydantic model class to structure the response

required
prompt str

The main prompt for the LLM

required
message_context list[dict]

Optional conversation context messages

None
max_retries int

Maximum number of retry attempts

2

Returns:

Name Type Description
dict dict[str, Any]

Validated structured data matching the Pydantic model

Raises:

Type Description
ValidationError

If the LLM response cannot be validated against the model

RuntimeError

If all retry attempts fail

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10298
10299
10300
10301
10302
10303
10304
10305
10306
10307
10308
10309
10310
10311
10312
10313
10314
10315
10316
10317
10318
10319
10320
10321
10322
10323
10324
10325
10326
10327
10328
10329
10330
10331
10332
10333
10334
10335
10336
10337
10338
10339
10340
10341
10342
10343
10344
10345
10346
10347
10348
10349
10350
10351
10352
10353
10354
10355
10356
10357
10358
10359
10360
10361
10362
10363
10364
10365
10366
10367
10368
10369
10370
10371
10372
10373
10374
10375
10376
10377
10378
10379
10380
10381
10382
10383
10384
10385
10386
10387
10388
10389
10390
10391
10392
10393
10394
10395
10396
10397
10398
10399
10400
10401
10402
10403
10404
10405
10406
10407
10408
10409
10410
10411
10412
10413
10414
10415
10416
10417
10418
10419
10420
10421
10422
10423
10424
10425
10426
10427
10428
10429
10430
10431
10432
10433
10434
10435
10436
10437
    async def a_format_class(self,
                             pydantic_model: type[BaseModel],
                             prompt: str,
                             message_context: list[dict] = None,
                             max_retries: int = 2, auto_context=True, session_id: str = None, **kwargs) -> dict[str, Any]:
        """
        State-of-the-art LLM-based structured data formatting using Pydantic models.

        Args:
            pydantic_model: The Pydantic model class to structure the response
            prompt: The main prompt for the LLM
            message_context: Optional conversation context messages
            max_retries: Maximum number of retry attempts

        Returns:
            dict: Validated structured data matching the Pydantic model

        Raises:
            ValidationError: If the LLM response cannot be validated against the model
            RuntimeError: If all retry attempts fail
        """

        if not LITELLM_AVAILABLE:
            raise RuntimeError("LiteLLM is required for structured formatting but not available")

        if session_id and self.active_session != session_id:
            self.active_session = session_id
        # Generate schema documentation
        schema = pydantic_model.model_json_schema() if issubclass(pydantic_model, BaseModel) else (json.loads(pydantic_model) if isinstance(pydantic_model, str) else pydantic_model)
        model_name = pydantic_model.__name__ if hasattr(pydantic_model, "__name__") else (pydantic_model.get("title", "UnknownModel") if isinstance(pydantic_model, dict) else "UnknownModel")

        # Create enhanced prompt with schema
        enhanced_prompt = f"""
    {prompt}

    CRITICAL FORMATTING REQUIREMENTS:
    1. Respond ONLY in valid YAML format
    2. Follow the exact schema structure provided
    3. Use appropriate data types (strings, lists, numbers, booleans)
    4. Include ALL required fields
    5. No additional comments, explanations, or text outside the YAML

    SCHEMA FOR {model_name}:
    {yaml.dump(schema, default_flow_style=False, indent=2)}

    EXAMPLE OUTPUT FORMAT:
    ```yaml
    # Your response here following the schema exactly
    field_name: "value"
    list_field:
      - "item1"
      - "item2"
    boolean_field: true
    number_field: 42
Respond in YAML format only:
"""
        # Prepare messages
        messages = []
        if message_context:
            messages.extend(message_context)
        messages.append({"role": "user", "content": enhanced_prompt})

        # Retry logic with progressive adjustments
        last_error = None

        for attempt in range(max_retries + 1):
            try:
                # Adjust parameters based on attempt
                temperature = 0.1 + (attempt * 0.1)  # Increase temperature slightly on retries
                max_tokens = min(2000 + (attempt * 500), 4000)  # Increase token limit on retries

                rprint(f"[{model_name}] Attempt {attempt + 1}/{max_retries + 1} (temp: {temperature})")

                # Generate LLM response
                response = await self.a_run_llm_completion(
                    model=self.amd.complex_llm_model,
                    messages=messages,
                    stream=False,
                    with_context=auto_context,
                    temperature=temperature,
                    max_tokens=max_tokens,
                    task_id=f"format_{model_name.lower()}_{attempt}"
                )

                if not response or not response.strip():
                    raise ValueError("Empty response from LLM")

                # Extract YAML content with multiple fallback strategies

                yaml_content = self._extract_yaml_content(response)


                if not yaml_content:
                    raise ValueError("No valid YAML content found in response")

                # Parse YAML
                try:
                    parsed_data = yaml.safe_load(yaml_content)
                except yaml.YAMLError as e:
                    raise ValueError(f"Invalid YAML syntax: {e}")
                iprint(parsed_data)
                if not isinstance(parsed_data, dict):
                    raise ValueError(f"Expected dict, got {type(parsed_data)}")

                # Validate against Pydantic model
                try:
                    if isinstance(pydantic_model, BaseModel):
                        validated_instance = pydantic_model.model_validate(parsed_data)
                        validated_data = validated_instance.model_dump()
                    else:
                        validated_data = parsed_data

                    rprint(f"✅ Successfully formatted {model_name} on attempt {attempt + 1}")
                    return validated_data

                except ValidationError as e:
                    detailed_errors = []
                    for error in e.errors():
                        field_path = " -> ".join(str(x) for x in error['loc'])
                        detailed_errors.append(f"Field '{field_path}': {error['msg']}")

                    error_msg = "Validation failed:\n" + "\n".join(detailed_errors)
                    raise ValueError(error_msg)

            except Exception as e:
                last_error = e
                wprint(f"[{model_name}] Attempt {attempt + 1} failed: {str(e)}")

                if attempt < max_retries:
                    # Add error feedback for next attempt
                    error_feedback = f"\n\nPREVIOUS ATTEMPT FAILED: {str(e)}\nPlease correct the issues and provide valid YAML matching the schema exactly."
                    messages[-1]["content"] = enhanced_prompt + error_feedback

                    # Brief delay before retry
                    # await asyncio.sleep(0.5 * (attempt + 1))
                else:
                    eprint(f"[{model_name}] All {max_retries + 1} attempts failed")

        # All attempts failed
        raise RuntimeError(f"Failed to format {model_name} after {max_retries + 1} attempts. Last error: {last_error}")
a_run(query, session_id='default', user_id=None, stream_callback=None, remember=True, **kwargs) async

Main entry point für Agent-Ausführung mit UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8441
8442
8443
8444
8445
8446
8447
8448
8449
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459
8460
8461
8462
8463
8464
8465
8466
8467
8468
8469
8470
8471
8472
8473
8474
8475
8476
8477
8478
8479
8480
8481
8482
8483
8484
8485
8486
8487
8488
8489
8490
8491
8492
8493
8494
8495
8496
8497
8498
8499
8500
8501
8502
8503
8504
8505
8506
8507
8508
8509
8510
8511
8512
8513
8514
8515
8516
8517
8518
8519
8520
8521
8522
8523
8524
8525
8526
8527
8528
8529
8530
8531
8532
8533
8534
8535
8536
8537
8538
8539
8540
8541
8542
8543
8544
8545
8546
8547
8548
8549
8550
8551
8552
8553
8554
8555
8556
8557
8558
8559
8560
8561
8562
8563
8564
8565
8566
8567
8568
8569
8570
8571
8572
8573
8574
8575
8576
8577
8578
8579
8580
8581
8582
8583
8584
8585
8586
8587
8588
async def a_run(
    self,
    query: str,
    session_id: str = "default",
    user_id: str = None,
    stream_callback: Callable = None,
    remember: bool = True,
    **kwargs
) -> str:
    """Main entry point für Agent-Ausführung mit UnifiedContextManager"""

    execution_start = self.progress_tracker.start_timer("total_execution")
    self.active_session = session_id
    result = None
    await self.progress_tracker.emit_event(ProgressEvent(
        event_type="execution_start",
        timestamp=time.time(),
        status=NodeStatus.RUNNING,
        node_name="FlowAgent",
        session_id=session_id,
        metadata={"query": query, "user_id": user_id}
    ))

    try:
        #Initialize or get session über UnifiedContextManager
        await self.initialize_session_context(session_id, max_history=200)

        #Store user message immediately in ChatSession wenn remember=True
        if remember:
            await self.context_manager.add_interaction(
                session_id,
                'user',
                query,
                metadata={"user_id": user_id}
            )

        # Set user context variables
        timestamp = datetime.now()
        self.variable_manager.register_scope('user', {
            'id': user_id,
            'session': session_id,
            'query': query,
            'timestamp': timestamp.isoformat()
        })

        # Update system variables
        self.variable_manager.set('system_context.timestamp', {'isoformat': timestamp.isoformat()})
        self.variable_manager.set('system_context.current_session', session_id)
        self.variable_manager.set('system_context.current_user', user_id)
        self.variable_manager.set('system_context.last_query', query)

        # Initialize with tool awareness
        await self.initialize_context_awareness()

        # VEREINFACHT: Prepare execution context - weniger Daten duplizieren
        self.shared.update({
            "current_query": query,
            "session_id": session_id,
            "user_id": user_id,
            "stream_callback": stream_callback,
            "remember": remember,
            # CENTRAL: Context Manager ist die primäre Context-Quelle
            "context_manager": self.context_manager,
            "variable_manager": self.variable_manager
        })

        # Set LLM models in shared context
        self.shared['fast_llm_model'] = self.amd.fast_llm_model
        self.shared['complex_llm_model'] = self.amd.complex_llm_model
        self.shared['persona_config'] = self.amd.persona
        self.shared['use_fast_response'] = self.amd.use_fast_response

        # Set system status
        self.shared["system_status"] = "running"
        self.is_running = True

        # Execute main orchestration flow
        result = await self._orchestrate_execution()

        #Store assistant response in ChatSession wenn remember=True
        if remember:
            await self.context_manager.add_interaction(
                session_id,
                'assistant',
                result,
                metadata={"user_id": user_id, "execution_duration": time.time() - execution_start}
            )

        total_duration = self.progress_tracker.end_timer("total_execution")

        await self.progress_tracker.emit_event(ProgressEvent(
            event_type="execution_complete",
            timestamp=time.time(),
            node_name="FlowAgent",
            status=NodeStatus.COMPLETED,
            node_duration=total_duration,
            session_id=session_id,
            metadata={
                "result_length": len(result),
                "summary": self.progress_tracker.get_summary(),
                "remembered": remember
            }
        ))

        # Checkpoint if needed
        if self.enable_pause_resume:
            with Spinner("Creating checkpoint..."):
                await self._maybe_checkpoint()
        return result

    except Exception as e:
        eprint(f"Agent execution failed: {e}", exc_info=True)
        error_response = f"I encountered an error: {str(e)}"
        result = error_response
        import traceback
        print(traceback.format_exc())

        # Store error in ChatSession wenn remember=True
        if remember:
            await self.context_manager.add_interaction(
                session_id,
                'assistant',
                error_response,
                metadata={
                    "user_id": user_id,
                    "error": True,
                    "error_type": type(e).__name__
                }
            )

        total_duration = self.progress_tracker.end_timer("total_execution")

        await self.progress_tracker.emit_event(ProgressEvent(
            event_type="error",
            timestamp=time.time(),
            node_name="FlowAgent",
            status=NodeStatus.FAILED,
            node_duration=total_duration,
            session_id=session_id,
            metadata={"error": str(e), "error_type": type(e).__name__}
        ))

        return error_response

    finally:
        self.shared["system_status"] = "idle"
        self.is_running = False
        self.active_session = None
a_run_with_format(query, response_format='frei-text', text_length='chat-conversation', custom_instructions='', **kwargs) async

Führe Agent mit spezifischem Format aus

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8694
8695
8696
8697
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707
8708
8709
8710
8711
8712
8713
8714
async def a_run_with_format(
    self,
    query: str,
    response_format: str = "frei-text",
    text_length: str = "chat-conversation",
    custom_instructions: str = "",
    **kwargs
) -> str:
    """Führe Agent mit spezifischem Format aus"""

    # Temporäre Format-Einstellung
    original_persona = self.amd.persona

    try:
        self.set_response_format(response_format, text_length, custom_instructions)
        response = await self.a_run(query, **kwargs)
        return response
    finally:
        # Restore original persona
        self.amd.persona = original_persona
        self.shared["persona_config"] = original_persona
add_custom_flow(flow, name)

Add a custom flow for dynamic execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10241
10242
10243
10244
def add_custom_flow(self, flow: AsyncFlow, name: str):
    """Add a custom flow for dynamic execution"""
    self.add_tool(flow.run_async, name=name, description=f"Custom flow: {flow.__class__.__name__}")
    rprint(f"Custom node added: {name}")
add_first_class_tool(tool_func, name, description)

Add a first-class meta-tool that can be used by the LLMReasonerNode. These are different from regular tools - they control agent sub-systems.

Parameters:

Name Type Description Default
tool_func Callable

The function to register as a meta-tool

required
name str

Name of the meta-tool

required
description str

Description of when and how to use it

required
Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10008
10009
10010
10011
10012
10013
10014
10015
10016
10017
10018
10019
10020
10021
10022
10023
10024
10025
10026
10027
10028
10029
10030
10031
10032
10033
10034
10035
10036
10037
10038
10039
10040
10041
10042
10043
10044
10045
10046
10047
10048
def add_first_class_tool(self, tool_func: Callable, name: str, description: str):
    """
    Add a first-class meta-tool that can be used by the LLMReasonerNode.
    These are different from regular tools - they control agent sub-systems.

    Args:
        tool_func: The function to register as a meta-tool
        name: Name of the meta-tool
        description: Description of when and how to use it
    """

    if not asyncio.iscoroutinefunction(tool_func):
        @wraps(tool_func)
        async def async_wrapper(*args, **kwargs):
            return await asyncio.to_thread(tool_func, *args, **kwargs)

        effective_func = async_wrapper
    else:
        effective_func = tool_func

    tool_name = name or effective_func.__name__
    tool_description = description or effective_func.__doc__ or "No description"

    # Validate the tool function
    if not callable(tool_func):
        raise ValueError("Tool function must be callable")

    # Register in the reasoner's meta-tool registry (if reasoner exists)
    if hasattr(self.task_flow, 'llm_reasoner'):
        if not hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
            self.task_flow.llm_reasoner.meta_tools_registry = {}

        self.task_flow.llm_reasoner.meta_tools_registry[tool_name] = {
            "function": effective_func,
            "description": tool_description,
            "args_schema": get_args_schema(tool_func)
        }

        rprint(f"First-class meta-tool added: {tool_name}")
    else:
        wprint("LLMReasonerNode not available for first-class tool registration")
add_tool(tool_func, name=None, description=None, is_new=False) async

Enhanced tool addition with intelligent analysis

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10050
10051
10052
10053
10054
10055
10056
10057
10058
10059
10060
10061
10062
10063
10064
10065
10066
10067
10068
10069
10070
10071
10072
10073
10074
10075
10076
10077
10078
10079
async def add_tool(self, tool_func: Callable, name: str = None, description: str = None, is_new=False):
    """Enhanced tool addition with intelligent analysis"""
    if not asyncio.iscoroutinefunction(tool_func):
        @wraps(tool_func)
        async def async_wrapper(*args, **kwargs):
            return await asyncio.to_thread(tool_func, *args, **kwargs)

        effective_func = async_wrapper
    else:
        effective_func = tool_func

    tool_name = name or effective_func.__name__
    tool_description = description or effective_func.__doc__ or "No description"

    # Store in registry
    self._tool_registry[tool_name] = {
        "function": effective_func,
        "description": tool_description,
        "args_schema": get_args_schema(tool_func)
    }

    # Add to available tools list
    if tool_name not in self.shared["available_tools"]:
        self.shared["available_tools"].append(tool_name)

    # Intelligent tool analysis
    if is_new:
        await self._analyze_tool_capabilities(tool_name, tool_description)

    rprint(f"Tool added with analysis: {tool_name}")
arun_function(function_name, *args, **kwargs) async

Asynchronously finds a function by its string name, executes it with the given arguments, and returns the result.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10250
10251
10252
10253
10254
10255
10256
10257
10258
10259
10260
10261
10262
10263
10264
10265
10266
10267
10268
10269
10270
10271
10272
10273
10274
10275
10276
10277
10278
10279
10280
10281
10282
10283
10284
10285
10286
10287
10288
10289
10290
10291
10292
10293
10294
async def arun_function(self, function_name: str, *args, **kwargs) -> Any:
    """
    Asynchronously finds a function by its string name, executes it with
    the given arguments, and returns the result.
    """
    rprint(f"Attempting to run function: {function_name} with args: {args}, kwargs: {kwargs}")
    target_function = self.get_tool_by_name(function_name)

    start_time = time.perf_counter()
    if not target_function:
        raise ValueError(f"Function '{function_name}' not found in the {self.amd.name}'s registered tools.")

    try:
        if asyncio.iscoroutinefunction(target_function):
            result = await target_function(*args, **kwargs)
        else:
            # If the function is not async, run it in a thread pool
            loop = asyncio.get_running_loop()
            result = await loop.run_in_executor(None, lambda: target_function(*args, **kwargs))

        if asyncio.iscoroutine(result):
            result = await result

        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="tool_call",  # Vereinheitlicht zu tool_call
                node_name="FlowAgent",
                status=NodeStatus.COMPLETED,
                success=True,
                duration=time.perf_counter() - start_time,
                tool_name=function_name,
                tool_args=kwargs,
                tool_result=result,
                is_meta_tool=False,  # Klarstellen, dass es kein Meta-Tool ist
                metadata={
                    "result_type": type(result).__name__,
                    "result_length": len(str(result))
                }
            ))
        rprint(f"Function {function_name} completed successfully with result: {result}")
        return result

    except Exception as e:
        eprint(f"Function {function_name} execution failed: {e}")
        raise
bind(*agents, shared_scopes=None, auto_sync=True)

Bind two or more agents together with shared and private variable spaces.

Parameters:

Name Type Description Default
*agents

FlowAgent instances to bind together

()
shared_scopes list[str]

List of scope names to share (default: ['world', 'results', 'system'])

None
auto_sync bool

Whether to automatically sync variables and context

True

Returns:

Name Type Description
dict

Binding configuration with agent references

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11462
11463
11464
11465
11466
11467
11468
11469
11470
11471
11472
11473
11474
11475
11476
11477
11478
11479
11480
11481
11482
11483
11484
11485
11486
11487
11488
11489
11490
11491
11492
11493
11494
11495
11496
11497
11498
11499
11500
11501
11502
11503
11504
11505
11506
11507
11508
11509
11510
11511
11512
11513
11514
11515
11516
11517
11518
11519
11520
11521
11522
11523
11524
11525
11526
11527
11528
11529
11530
11531
11532
11533
11534
11535
11536
11537
11538
11539
11540
11541
11542
11543
11544
11545
11546
11547
11548
11549
11550
11551
11552
11553
11554
11555
11556
11557
11558
11559
11560
11561
11562
11563
11564
11565
11566
11567
11568
11569
11570
11571
11572
11573
11574
11575
11576
11577
11578
11579
11580
11581
11582
11583
11584
11585
11586
11587
11588
11589
11590
def bind(self, *agents, shared_scopes: list[str] = None, auto_sync: bool = True):
    """
    Bind two or more agents together with shared and private variable spaces.

    Args:
        *agents: FlowAgent instances to bind together
        shared_scopes: List of scope names to share (default: ['world', 'results', 'system'])
        auto_sync: Whether to automatically sync variables and context

    Returns:
        dict: Binding configuration with agent references
    """
    if shared_scopes is None:
        shared_scopes = ['world', 'results', 'system']

    # Create unique binding ID
    binding_id = f"bind_{int(time.time())}_{len(agents)}"

    # All agents in this binding (including self)
    all_agents = [self] + list(agents)

    # Create shared variable manager that all agents will reference
    shared_world_model = {}
    shared_state = {}

    # Merge existing data from all agents
    for agent in all_agents:
        # Merge world models
        shared_world_model.update(agent.world_model)
        shared_state.update(agent.shared)

    # Create shared variable manager
    shared_variable_manager = VariableManager(shared_world_model, shared_state)

    # Set up shared scopes with merged data
    for scope_name in shared_scopes:
        merged_scope = {}
        for agent in all_agents:
            if hasattr(agent, 'variable_manager') and agent.variable_manager:
                agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                if isinstance(agent_scope_data, dict):
                    merged_scope.update(agent_scope_data)
        shared_variable_manager.register_scope(scope_name, merged_scope)

    # Create binding configuration
    binding_config = {
        'binding_id': binding_id,
        'agents': all_agents,
        'shared_scopes': shared_scopes,
        'auto_sync': auto_sync,
        'shared_variable_manager': shared_variable_manager,
        'private_managers': {},
        'created_at': datetime.now().isoformat()
    }

    # Configure each agent
    for i, agent in enumerate(all_agents):
        agent_private_id = f"{binding_id}_agent_{i}_{agent.amd.name}"

        # Create private variable manager for agent-specific data
        private_world_model = agent.world_model.copy()
        private_shared = agent.shared.copy()
        private_variable_manager = VariableManager(private_world_model, private_shared)

        # Set up private scopes (user, session-specific data, agent-specific configs)
        private_scopes = ['user', 'agent', 'session_private', 'tasks_private']
        for scope_name in private_scopes:
            if hasattr(agent, 'variable_manager') and agent.variable_manager:
                agent_scope_data = agent.variable_manager.scopes.get(scope_name, {})
                private_variable_manager.register_scope(f"{scope_name}_{agent.amd.name}", agent_scope_data)

        binding_config['private_managers'][agent.amd.name] = private_variable_manager

        # Replace agent's variable manager with a unified one
        unified_manager = UnifiedBindingManager(
            shared_manager=shared_variable_manager,
            private_manager=private_variable_manager,
            agent_name=agent.amd.name,
            shared_scopes=shared_scopes,
            auto_sync=auto_sync,
            binding_config=binding_config
        )

        # Store original managers for unbinding
        if not hasattr(agent, '_original_managers'):
            agent._original_managers = {
                'variable_manager': agent.variable_manager,
                'world_model': agent.world_model.copy(),
                'shared': agent.shared.copy()
            }

        # Set new unified manager
        agent.variable_manager = unified_manager
        agent.world_model = shared_world_model
        agent.shared = shared_state

        # Update shared state with binding info
        agent.shared['binding_config'] = binding_config
        agent.shared['is_bound'] = True
        agent.shared['binding_id'] = binding_id
        agent.shared['bound_agents'] = [a.amd.name for a in all_agents]

        # Sync context manager if available
        if hasattr(agent, 'context_manager') and agent.context_manager:
            agent.context_manager.variable_manager = unified_manager

            # Share session managers between bound agents if auto_sync is enabled
            if auto_sync:
                # Merge session managers from all agents
                all_sessions = {}
                for bound_agent in all_agents:
                    if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                        if hasattr(bound_agent.context_manager, 'session_managers'):
                            all_sessions.update(bound_agent.context_manager.session_managers)

                # Update all agents with merged sessions
                for bound_agent in all_agents:
                    if hasattr(bound_agent, 'context_manager') and bound_agent.context_manager:
                        bound_agent.context_manager.session_managers.update(all_sessions)

    # Set up auto-sync mechanism if enabled
    if auto_sync:
        binding_config['sync_handler'] = BindingSyncHandler(binding_config)

    rprint(f"Successfully bound {len(all_agents)} agents together (Binding ID: {binding_id})")
    rprint(f"Shared scopes: {', '.join(shared_scopes)}")
    rprint(f"Bound agents: {', '.join([agent.amd.name for agent in all_agents])}")

    return binding_config
clean_memory(deep_clean=False) async

Clean memory and context of the agent

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10642
10643
10644
10645
10646
10647
10648
10649
10650
10651
10652
10653
10654
10655
10656
10657
10658
10659
10660
10661
10662
10663
10664
10665
10666
10667
10668
10669
10670
10671
10672
10673
10674
10675
10676
10677
10678
10679
10680
10681
10682
10683
10684
10685
10686
10687
10688
10689
10690
10691
10692
10693
10694
10695
10696
10697
10698
10699
10700
10701
10702
10703
10704
10705
10706
10707
10708
10709
10710
10711
10712
async def clean_memory(self, deep_clean: bool = False) -> bool:
    """Clean memory and context of the agent"""
    try:
        # Clear current context first
        self.clear_context()

        # Clean world model
        self.shared["world_model"] = {}
        self.world_model = {}

        # Clean performance metrics
        self.shared["performance_metrics"] = {}

        # Deep clean session storage
        session_managers = self.shared.get("session_managers", {})
        if session_managers:
            for _manager_name, manager in session_managers.items():
                if hasattr(manager, 'clear_all_history'):
                    await manager.clear_all_history()
                elif hasattr(manager, 'clear_history'):
                    manager.clear_history()

        # Clear session managers entirely
        self.shared["session_managers"] = {}
        self.shared["session_initialized"] = False

        # Clean variable manager completely
        if hasattr(self, 'variable_manager'):
            # Reinitialize with clean state
            self.variable_manager = VariableManager({}, self.shared)
            self._setup_variable_scopes()

        # Clean tool analysis cache if deep clean
        if deep_clean:
            self._tool_capabilities = {}
            self._tool_analysis_cache = {}

            # Remove tool analysis file
            if hasattr(self, 'tool_analysis_file') and os.path.exists(self.tool_analysis_file):
                try:
                    os.remove(self.tool_analysis_file)
                    rprint("Removed tool analysis cache file")
                except:
                    pass

        # Clean checkpoint data
        self.checkpoint_data = {}
        self.last_checkpoint = None

        # Clean execution history
        if hasattr(self.task_flow, 'executor_node'):
            self.task_flow.executor_node.execution_history = []
            self.task_flow.executor_node.results_store = {}

        # Clean context manager sessions
        if hasattr(self.task_flow, 'context_manager'):
            self.task_flow.context_manager.session_managers = {}

        # Clean LLM call statistics
        self.shared.pop("llm_call_stats", None)

        # Force garbage collection
        import gc
        gc.collect()

        rprint(f"Memory cleaned (deep_clean: {deep_clean})")
        return True

    except Exception as e:
        eprint(f"Failed to clean memory: {e}")
        return False
cleanup_session_context(session_id=None, keep_count=100, remove_old_snapshots=True) async

Cleanup session context by removing old snapshots and entries

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9712
9713
9714
9715
9716
9717
9718
9719
9720
9721
9722
9723
9724
9725
9726
9727
9728
9729
9730
9731
9732
9733
9734
9735
9736
9737
9738
9739
9740
9741
9742
9743
9744
9745
9746
9747
9748
9749
9750
9751
9752
9753
9754
9755
9756
9757
9758
9759
9760
9761
9762
9763
9764
9765
9766
9767
9768
9769
9770
9771
9772
9773
9774
9775
9776
9777
9778
9779
9780
9781
9782
9783
9784
9785
async def cleanup_session_context(self, session_id: str = None, keep_count: int = 100,
                                  remove_old_snapshots: bool = True) -> dict[str, Any]:
    """Cleanup session context by removing old snapshots and entries"""
    try:
        session_id = session_id or self.shared.get("session_id", "default")

        if not self.context_manager:
            return {"error": "Context manager not available"}

        session = self.context_manager.session_managers.get(session_id)
        if not session or not hasattr(session, 'history'):
            return {"error": f"Session {session_id} not found or has no history"}

        cleanup_stats = {
            "original_message_count": len(session.history),
            "context_snapshots_removed": 0,
            "context_entries_removed": 0,
            "regular_messages_kept": 0,
            "cleanup_performed": False
        }

        if len(session.history) <= keep_count:
            return {**cleanup_stats, "message": "No cleanup needed"}

        # Separate different types of messages
        regular_messages = []
        context_snapshots = []
        context_entries = []

        for message in session.history:
            metadata = message.get("metadata", {})

            if metadata.get("is_context_snapshot"):
                context_snapshots.append(message)
            elif metadata.get("is_context_entry"):
                context_entries.append(message)
            else:
                regular_messages.append(message)

        # Keep most recent regular messages
        messages_to_keep = regular_messages[-keep_count:]
        cleanup_stats["regular_messages_kept"] = len(messages_to_keep)

        # Keep most recent context snapshots (if not removing)
        if not remove_old_snapshots:
            recent_snapshots = context_snapshots[-5:]  # Keep last 5 snapshots
            messages_to_keep.extend(recent_snapshots)
        else:
            cleanup_stats["context_snapshots_removed"] = len(context_snapshots)

        # Keep persistent context entries
        persistent_entries = [
            entry for entry in context_entries
            if entry.get("persistent", True)
        ]
        messages_to_keep.extend(persistent_entries)
        cleanup_stats["context_entries_removed"] = len(context_entries) - len(persistent_entries)

        # Sort by timestamp and update session
        messages_to_keep.sort(key=lambda x: x.get("timestamp", ""))
        session.history = messages_to_keep

        cleanup_stats.update({
            "final_message_count": len(session.history),
            "cleanup_performed": True,
            "messages_removed": cleanup_stats["original_message_count"] - len(session.history)
        })

        rprint(f"Session cleanup completed: {cleanup_stats['messages_removed']} messages removed")
        return cleanup_stats

    except Exception as e:
        eprint(f"Failed to cleanup session context: {e}")
        return {"error": str(e)}
clear_context(session_id=None)

Clear context über UnifiedContextManager mit Session-spezifischer Unterstützung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10564
10565
10566
10567
10568
10569
10570
10571
10572
10573
10574
10575
10576
10577
10578
10579
10580
10581
10582
10583
10584
10585
10586
10587
10588
10589
10590
10591
10592
10593
10594
10595
10596
10597
10598
10599
10600
10601
10602
10603
10604
10605
10606
10607
10608
10609
10610
10611
10612
10613
10614
10615
10616
10617
10618
10619
10620
10621
10622
10623
10624
10625
10626
10627
10628
10629
10630
10631
10632
10633
10634
10635
10636
10637
10638
10639
10640
def clear_context(self, session_id: str = None) -> bool:
    """Clear context über UnifiedContextManager mit Session-spezifischer Unterstützung"""
    try:
        #Clear über Context Manager
        if session_id:
            # Clear specific session
            if session_id in self.context_manager.session_managers:
                session = self.context_manager.session_managers[session_id]
                if hasattr(session, 'history'):
                    session.history = []
                elif isinstance(session, dict) and 'history' in session:
                    session['history'] = []

                # Remove from session managers
                del self.context_manager.session_managers[session_id]

                # Clear variable manager scope for this session
                if self.variable_manager:
                    scope_name = f'session_{session_id}'
                    if scope_name in self.variable_manager.scopes:
                        del self.variable_manager.scopes[scope_name]

                rprint(f"Context cleared for session: {session_id}")
        else:
            # Clear all sessions
            for session_id, session in self.context_manager.session_managers.items():
                if hasattr(session, 'history'):
                    session.history = []
                elif isinstance(session, dict) and 'history' in session:
                    session['history'] = []

            self.context_manager.session_managers = {}
            rprint("Context cleared for all sessions")

        # Clear context cache
        self.context_manager._invalidate_cache(session_id)

        # Clear current execution context in shared
        context_keys_to_clear = [
            "current_query", "current_response", "current_plan", "tasks",
            "results", "task_plans", "session_data", "formatted_context",
            "synthesized_response", "quality_assessment", "plan_adaptations",
            "executor_performance", "llm_tool_conversation", "aggregated_context"
        ]

        for key in context_keys_to_clear:
            if key in self.shared:
                if isinstance(self.shared[key], dict):
                    self.shared[key] = {}
                elif isinstance(self.shared[key], list):
                    self.shared[key] = []
                else:
                    self.shared[key] = None

        # Clear variable manager scopes (except core system variables)
        if hasattr(self, 'variable_manager'):
            # Clear user, results, tasks scopes
            self.variable_manager.register_scope('user', {})
            self.variable_manager.register_scope('results', {})
            self.variable_manager.register_scope('tasks', {})
            # Reset cache
            self.variable_manager._cache.clear()

        # Reset execution state
        self.is_running = False
        self.is_paused = False
        self.shared["system_status"] = "idle"

        # Clear progress tracking
        if hasattr(self, 'progress_tracker'):
            self.progress_tracker.reset_session_metrics()

        return True

    except Exception as e:
        eprint(f"Failed to clear context: {e}")
        return False
close() async

Clean shutdown

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10714
10715
10716
10717
10718
10719
10720
10721
10722
10723
10724
10725
10726
10727
10728
10729
10730
10731
10732
10733
10734
10735
10736
10737
async def close(self):
    """Clean shutdown"""
    self.is_running = False
    self._shutdown_event.set()

    # Create final checkpoint
    if self.enable_pause_resume:
        checkpoint = await self._create_checkpoint()
        await self._save_checkpoint(checkpoint, "final_checkpoint.pkl")

    # Shutdown executor
    self.executor.shutdown(wait=True)

    # Close servers
    if self.a2a_server:
        await self.a2a_server.close()

    if self.mcp_server:
        await self.mcp_server.close()

    if hasattr(self, '_mcp_session_manager'):
        await self._mcp_session_manager.cleanup_all()

    rprint("Agent shutdown complete")
configure_persona_integration(apply_method='system_prompt', integration_level='light')

Configure how persona is applied

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9008
9009
9010
9011
9012
9013
9014
9015
def configure_persona_integration(self, apply_method: str = "system_prompt", integration_level: str = "light"):
    """Configure how persona is applied"""
    if self.amd.persona:
        self.amd.persona.apply_method = apply_method
        self.amd.persona.integration_level = integration_level
        rprint(f"Persona integration updated: {apply_method}, {integration_level}")
    else:
        wprint("No persona configured to update")
delete_old_checkpoints(keep_count=5, max_age_hours=168) async

Delete old checkpoints, keeping the most recent ones

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9931
9932
9933
9934
9935
9936
9937
9938
9939
9940
9941
9942
9943
9944
9945
9946
9947
9948
9949
9950
9951
9952
9953
9954
9955
9956
9957
9958
9959
9960
9961
9962
9963
9964
9965
9966
9967
9968
9969
9970
9971
9972
9973
9974
9975
9976
9977
9978
9979
9980
9981
9982
9983
9984
9985
9986
9987
9988
async def delete_old_checkpoints(self, keep_count: int = 5, max_age_hours: int = 168) -> dict[str, Any]:
    """Delete old checkpoints, keeping the most recent ones"""
    try:
        checkpoints = self.list_available_checkpoints(
            max_age_hours=max_age_hours * 2)  # Look further back for deletion

        deleted_count = 0
        deleted_size_kb = 0
        errors = []

        if len(checkpoints) > keep_count:
            # Keep the newest, delete the rest (except final checkpoint)
            to_delete = checkpoints[keep_count:]

            for checkpoint in to_delete:
                if checkpoint["checkpoint_type"] != "final":  # Never delete final checkpoint
                    try:
                        os.remove(checkpoint["filepath"])
                        deleted_count += 1
                        deleted_size_kb += checkpoint["file_size_kb"]
                        rprint(f"Deleted old checkpoint: {checkpoint['filename']}")
                    except Exception as e:
                        import traceback
                        print(traceback.format_exc())
                        errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

        # Also delete checkpoints older than max_age_hours
        old_checkpoints = [cp for cp in checkpoints if
                           cp["age_hours"] > max_age_hours and cp["checkpoint_type"] != "final"]
        for checkpoint in old_checkpoints:
            if checkpoint not in checkpoints[keep_count:]:  # Don't double-delete
                try:
                    os.remove(checkpoint["filepath"])
                    deleted_count += 1
                    deleted_size_kb += checkpoint["file_size_kb"]
                    rprint(f"Deleted aged checkpoint: {checkpoint['filename']}")
                except Exception as e:
                    import traceback
                    print(traceback.format_exc())
                    errors.append(f"Failed to delete {checkpoint['filename']}: {e}")

        return {
            "success": True,
            "deleted_count": deleted_count,
            "freed_space_kb": round(deleted_size_kb, 1),
            "remaining_checkpoints": len(checkpoints) - deleted_count,
            "errors": errors
        }

    except Exception as e:
        import traceback
        print(traceback.format_exc())
        eprint(f"Failed to delete old checkpoints: {e}")
        return {
            "success": False,
            "error": str(e),
            "deleted_count": 0
        }
explain_reasoning_process() async

Erkläre den Reasoning-Prozess des Agenten

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9157
9158
9159
9160
9161
9162
9163
9164
9165
9166
9167
9168
9169
9170
9171
9172
9173
9174
9175
9176
9177
9178
9179
9180
9181
9182
9183
9184
9185
9186
9187
9188
9189
9190
9191
9192
9193
9194
9195
9196
9197
9198
9199
9200
9201
    async def explain_reasoning_process(self) -> str:
        """Erkläre den Reasoning-Prozess des Agenten"""
        if not LITELLM_AVAILABLE:
            return "Reasoning explanation requires LLM capabilities."

        summary = await self.get_task_execution_summary()

        prompt = f"""
Erkläre den Reasoning-Prozess dieses AI-Agenten in verständlicher Form:

## Ausführungszusammenfassung
- Total Tasks: {summary['total_tasks']}
- Erfolgreich: {len(summary['completed_tasks'])}
- Fehlgeschlagen: {len(summary['failed_tasks'])}
- Plan-Adaptationen: {summary['adaptations']}
- Verwendete Tools: {', '.join(set(summary['tools_used']))}
- Task-Typen: {summary['task_types_used']}

## Task-Details
Erfolgreiche Tasks:
{self._format_tasks_for_explanation(summary['completed_tasks'])}

## Anweisungen
Erkläre in 2-3 Absätzen:
1. Welche Strategie der Agent gewählt hat
2. Wie er die Aufgabe in Tasks unterteilt hat
3. Wie er auf unerwartete Ergebnisse reagiert hat (falls Adaptationen)
4. Was die wichtigsten Erkenntnisse waren

Schreibe für einen technischen Nutzer, aber verständlich."""

        try:
            response = await self.a_run_llm_completion(
                model=self.amd.complex_llm_model,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.5,
                max_tokens=800,task_id="reasoning_explanation"
            )

            return response

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            return f"Could not generate reasoning explanation: {e}"
format_text(text, **context)

Format text with variables

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8798
8799
8800
def format_text(self, text: str, **context) -> str:
    """Format text with variables"""
    return self.variable_manager.format_text(text, context)
get_available_formats()

Erhalte verfügbare Format- und Längen-Optionen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8679
8680
8681
8682
8683
8684
8685
8686
8687
8688
8689
8690
8691
8692
def get_available_formats(self) -> dict[str, list[str]]:
    """Erhalte verfügbare Format- und Längen-Optionen"""
    return {
        "formats": [f.value for f in ResponseFormat],
        "lengths": [l.value for l in TextLength],
        "format_descriptions": {
            f.value: FormatConfig(response_format=f).get_format_instructions()
            for f in ResponseFormat
        },
        "length_descriptions": {
            l.value: FormatConfig(text_length=l).get_length_instructions()
            for l in TextLength
        }
    }
get_available_variables()

Get available variables for dynamic formatting

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9017
9018
9019
def get_available_variables(self) -> dict[str, str]:
    """Get available variables for dynamic formatting"""
    return self.variable_manager.get_available_variables()
get_context(session_id=None, format_for_llm=True) async

ÜBERARBEITET: Get context über UnifiedContextManager statt verteilte Quellen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8916
8917
8918
8919
8920
8921
8922
8923
8924
8925
8926
8927
8928
8929
8930
8931
8932
8933
8934
8935
8936
8937
8938
8939
8940
8941
8942
8943
8944
8945
8946
async def get_context(self, session_id: str = None, format_for_llm: bool = True) -> str | dict[str, Any]:
    """
    ÜBERARBEITET: Get context über UnifiedContextManager statt verteilte Quellen
    """
    try:
        session_id = session_id or self.shared.get("session_id", self.active_session)
        query = self.shared.get("current_query", "")

        #Hole unified context über Context Manager
        unified_context = await self.context_manager.build_unified_context(session_id, query, "full")


        if format_for_llm:
            return self.context_manager.get_formatted_context_for_llm(unified_context)
        else:
            return unified_context

    except Exception as e:
        import traceback
        print(traceback.format_exc())
        eprint(f"Failed to generate context via UnifiedContextManager: {e}")

        # FALLBACK: Fallback zu alter Methode falls UnifiedContextManager fehlschlägt
        if format_for_llm:
            return f"Error generating context: {str(e)}"
        else:
            return {
                "error": str(e),
                "generated_at": datetime.now().isoformat(),
                "fallback_mode": True
            }
get_context_overview(session_id=None, display=False) async

Detaillierte Übersicht des aktuellen Contexts mit Token-Counts und optionaler Display-Darstellung

Parameters:

Name Type Description Default
session_id str

Session ID für context (default: active_session)

None
display bool

Ob die Übersicht im Terminal-Style angezeigt werden soll

False

Returns:

Name Type Description
dict dict[str, Any]

Detaillierte Context-Übersicht mit Raw-Daten und Token-Counts

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10746
10747
10748
10749
10750
10751
10752
10753
10754
10755
10756
10757
10758
10759
10760
10761
10762
10763
10764
10765
10766
10767
10768
10769
10770
10771
10772
10773
10774
10775
10776
10777
10778
10779
10780
10781
10782
10783
10784
10785
10786
10787
10788
10789
10790
10791
10792
10793
10794
10795
10796
10797
10798
10799
10800
10801
10802
10803
10804
10805
10806
10807
10808
10809
10810
10811
10812
10813
10814
10815
10816
10817
10818
10819
10820
10821
10822
10823
10824
10825
10826
10827
10828
10829
10830
10831
10832
10833
10834
10835
10836
10837
10838
10839
10840
10841
10842
10843
10844
10845
10846
10847
10848
10849
10850
10851
10852
10853
10854
10855
10856
10857
10858
10859
10860
10861
10862
10863
10864
10865
10866
10867
10868
10869
10870
10871
10872
10873
10874
10875
10876
10877
10878
10879
10880
10881
10882
10883
10884
10885
10886
10887
10888
10889
10890
10891
10892
10893
10894
10895
10896
10897
10898
10899
10900
10901
10902
10903
10904
10905
10906
10907
10908
10909
10910
10911
10912
10913
10914
10915
10916
10917
10918
10919
10920
10921
10922
10923
10924
10925
10926
10927
10928
10929
10930
10931
10932
10933
10934
10935
10936
10937
10938
10939
10940
10941
10942
10943
10944
10945
10946
10947
10948
10949
10950
10951
10952
10953
10954
10955
10956
10957
10958
10959
10960
10961
10962
10963
10964
10965
10966
10967
10968
10969
10970
10971
10972
10973
10974
10975
10976
10977
10978
10979
10980
10981
10982
10983
10984
10985
10986
10987
10988
10989
10990
10991
10992
10993
10994
10995
10996
10997
10998
10999
11000
11001
11002
11003
11004
11005
11006
11007
11008
11009
11010
11011
11012
11013
11014
11015
11016
11017
11018
11019
11020
11021
11022
11023
11024
11025
11026
async def get_context_overview(self, session_id: str = None, display: bool = False) -> dict[str, Any]:
    """
    Detaillierte Übersicht des aktuellen Contexts mit Token-Counts und optionaler Display-Darstellung

    Args:
        session_id: Session ID für context (default: active_session)
        display: Ob die Übersicht im Terminal-Style angezeigt werden soll

    Returns:
        dict: Detaillierte Context-Übersicht mit Raw-Daten und Token-Counts
    """
    try:
        session_id = session_id or self.active_session or "default"

        # Token counting function
        def count_tokens(text: str) -> int:
            """Einfache Token-Approximation (4 chars ≈ 1 token für deutsche/englische Texte)"""
            try:
                from litellm import token_counter
                return token_counter(self.amd.fast_llm_model, text=text)
            except:
                pass
            return max(1, len(str(text)) // 4)

        context_overview = {
            "session_info": {
                "session_id": session_id,
                "agent_name": self.amd.name,
                "timestamp": datetime.now().isoformat(),
                "active_session": self.active_session,
                "is_running": self.is_running
            },
            "system_prompt": {},
            "meta_tools": {},
            "agent_tools": {},
            "mcp_tools": {},
            "variables": {},
            "system_history": {},
            "unified_context": {},
            "reasoning_context": {},
            "llm_tool_context": {},
            "token_summary": {}
        }

        # === SYSTEM PROMPT ANALYSIS ===
        system_message = self.amd.get_system_message_with_persona()
        context_overview["system_prompt"] = {
            "raw_data": system_message,
            "token_count": count_tokens(system_message),
            "components": {
                "base_message": self.amd.system_message,
                "persona_active": self.amd.persona is not None,
                "persona_name": self.amd.persona.name if self.amd.persona else None,
                "persona_integration": self.amd.persona.apply_method if self.amd.persona else None
            }
        }

        # === META TOOLS ANALYSIS ===
        if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'meta_tools_registry'):
            meta_tools = self.task_flow.llm_reasoner.meta_tools_registry
        else:
            meta_tools = {}

        meta_tools_info = ""
        for tool_name, tool_info in meta_tools.items():
            tool_desc = tool_info.get("description", "No description")
            meta_tools_info += f"{tool_name}: {tool_desc}\n"

        # Standard Meta-Tools
        standard_meta_tools = [
            "internal_reasoning", "manage_internal_task_stack", "delegate_to_llm_tool_node",
            "create_and_execute_plan", "advance_outline_step", "write_to_variables",
            "read_from_variables", "direct_response"
        ]

        for meta_tool in standard_meta_tools:
            meta_tools_info += f"{meta_tool}: Built-in meta-tool for agent orchestration\n"

        context_overview["meta_tools"] = {
            "raw_data": meta_tools_info,
            "token_count": count_tokens(meta_tools_info),
            "count": len(meta_tools) + len(standard_meta_tools),
            "custom_meta_tools": list(meta_tools.keys()),
            "standard_meta_tools": standard_meta_tools
        }

        # === AGENT TOOLS ANALYSIS ===
        tools_info = ""
        tool_capabilities_text = ""

        for tool_name in self.shared.get("available_tools", []):
            tool_data = self._tool_registry.get(tool_name, {})
            description = tool_data.get("description", "No description")
            args_schema = tool_data.get("args_schema", "()")
            tools_info += f"{tool_name}{args_schema}: {description}\n"

            # Tool capabilities if available
            if tool_name in self._tool_capabilities:
                cap = self._tool_capabilities[tool_name]
                primary_function = cap.get("primary_function", "Unknown")
                use_cases = cap.get("use_cases", [])
                tool_capabilities_text += f"{tool_name}: {primary_function}\n"
                if use_cases:
                    tool_capabilities_text += f"  Use cases: {', '.join(use_cases[:3])}\n"

        context_overview["agent_tools"] = {
            "raw_data": tools_info,
            "capabilities_data": tool_capabilities_text,
            "token_count": count_tokens(tools_info + tool_capabilities_text),
            "count": len(self.shared.get("available_tools", [])),
            "analyzed_count": len(self._tool_capabilities),
            "tool_names": self.shared.get("available_tools", []),
            "intelligence_level": "high" if self._tool_capabilities else "basic"
        }

        # === MCP TOOLS ANALYSIS ===
        # Placeholder für MCP Tools (falls implementiert)
        mcp_tools_info = "No MCP tools currently active"
        if self.mcp_server:
            mcp_tools_info = f"MCP Server active: {getattr(self.mcp_server, 'name', 'Unknown')}"

        context_overview["mcp_tools"] = {
            "raw_data": mcp_tools_info,
            "token_count": count_tokens(mcp_tools_info),
            "server_active": bool(self.mcp_server),
            "server_name": getattr(self.mcp_server, 'name', None) if self.mcp_server else None
        }

        # === VARIABLES ANALYSIS ===
        variables_text = ""
        if self.variable_manager:
            variables_text = self.variable_manager.get_llm_variable_context()
        else:
            variables_text = "No variable manager available"

        context_overview["variables"] = {
            "raw_data": variables_text,
            "token_count": count_tokens(variables_text),
            "manager_available": bool(self.variable_manager),
            "total_scopes": len(self.variable_manager.scopes) if self.variable_manager else 0,
            "scope_names": list(self.variable_manager.scopes.keys()) if self.variable_manager else []
        }

        # === SYSTEM HISTORY ANALYSIS ===
        history_text = ""
        if self.context_manager and session_id in self.context_manager.session_managers:
            session = self.context_manager.session_managers[session_id]
            if hasattr(session, 'history'):
                history_count = len(session.history)
                history_text = f"Session History: {history_count} messages\n"

                # Recent messages preview
                for msg in session.history[-3:]:
                    role = msg.get('role', 'unknown')
                    content = msg.get('content', '')[:100] + "..." if len(
                        msg.get('content', '')) > 100 else msg.get('content', '')
                    timestamp = msg.get('timestamp', '')[:19]
                    history_text += f"[{timestamp}] {role}: {content}\n"
            elif isinstance(session, dict) and 'history' in session:
                history_count = len(session['history'])
                history_text = f"Fallback Session History: {history_count} messages"
        else:
            history_text = "No session history available"

        context_overview["system_history"] = {
            "raw_data": history_text,
            "token_count": count_tokens(history_text),
            "session_initialized": self.shared.get("session_initialized", False),
            "context_manager_available": bool(self.context_manager),
            "session_count": len(self.context_manager.session_managers) if self.context_manager else 0
        }

        # === UNIFIED CONTEXT ANALYSIS ===
        unified_context_text = ""
        try:
            unified_context = await self.context_manager.build_unified_context(session_id, "",
                                                                               "full") if self.context_manager else {}
            if unified_context:
                formatted_context = self.context_manager.get_formatted_context_for_llm(unified_context)
                unified_context_text = formatted_context
            else:
                unified_context_text = "No unified context available"
        except Exception as e:
            unified_context_text = f"Error building unified context: {str(e)}"

        context_overview["unified_context"] = {
            "raw_data": unified_context_text,
            "token_count": count_tokens(unified_context_text),
            "build_successful": "Error" not in unified_context_text,
            "manager_available": bool(self.context_manager)
        }

        # === REASONING CONTEXT ANALYSIS ===
        reasoning_context_text = ""
        if hasattr(self.task_flow, 'llm_reasoner') and hasattr(self.task_flow.llm_reasoner, 'reasoning_context'):
            reasoning_context = self.task_flow.llm_reasoner.reasoning_context
            reasoning_context_text = f"Reasoning Context: {len(reasoning_context)} entries\n"

            # Recent reasoning entries
            for entry in reasoning_context[-3:]:
                entry_type = entry.get('type', 'unknown')
                content = str(entry.get('content', ''))[:150] + "..." if len(
                    str(entry.get('content', ''))) > 150 else str(entry.get('content', ''))
                reasoning_context_text += f"  {entry_type}: {content}\n"
        else:
            reasoning_context_text = "No reasoning context available"

        context_overview["reasoning_context"] = {
            "raw_data": reasoning_context_text,
            "token_count": count_tokens(reasoning_context_text),
            "reasoner_available": hasattr(self.task_flow, 'llm_reasoner'),
            "context_entries": len(self.task_flow.llm_reasoner.reasoning_context) if hasattr(self.task_flow,
                                                                                             'llm_reasoner') and hasattr(
                self.task_flow.llm_reasoner, 'reasoning_context') else 0
        }

        # === LLM TOOL CONTEXT ANALYSIS ===
        llm_tool_context_text = ""
        if hasattr(self.task_flow, 'llm_tool_node'):
            llm_tool_context_text = f"LLM Tool Node available with max {self.task_flow.llm_tool_node.max_tool_calls} tool calls\n"
            if hasattr(self.task_flow.llm_tool_node, 'call_log'):
                call_log = self.task_flow.llm_tool_node.call_log
                llm_tool_context_text += f"Call log: {len(call_log)} entries\n"
        else:
            llm_tool_context_text = "No LLM Tool Node available"

        context_overview["llm_tool_context"] = {
            "raw_data": llm_tool_context_text,
            "token_count": count_tokens(llm_tool_context_text),
            "node_available": hasattr(self.task_flow, 'llm_tool_node'),
            "max_tool_calls": getattr(self.task_flow.llm_tool_node, 'max_tool_calls', 0) if hasattr(self.task_flow,
                                                                                                    'llm_tool_node') else 0
        }

        # === TOKEN SUMMARY ===
        total_tokens = sum([
            context_overview["system_prompt"]["token_count"],
            context_overview["meta_tools"]["token_count"],
            context_overview["agent_tools"]["token_count"],
            context_overview["mcp_tools"]["token_count"],
            context_overview["variables"]["token_count"],
            context_overview["system_history"]["token_count"],
            context_overview["unified_context"]["token_count"],
            context_overview["reasoning_context"]["token_count"],
            context_overview["llm_tool_context"]["token_count"]
        ])

        context_overview["token_summary"] = {
            "total_tokens": total_tokens,
            "breakdown": {
                "system_prompt": context_overview["system_prompt"]["token_count"],
                "meta_tools": context_overview["meta_tools"]["token_count"],
                "agent_tools": context_overview["agent_tools"]["token_count"],
                "mcp_tools": context_overview["mcp_tools"]["token_count"],
                "variables": context_overview["variables"]["token_count"],
                "system_history": context_overview["system_history"]["token_count"],
                "unified_context": context_overview["unified_context"]["token_count"],
                "reasoning_context": context_overview["reasoning_context"]["token_count"],
                "llm_tool_context": context_overview["llm_tool_context"]["token_count"]
            },
            "percentage_breakdown": {}
        }

        # Calculate percentages
        for component, token_count in context_overview["token_summary"]["breakdown"].items():
            percentage = (token_count / total_tokens * 100) if total_tokens > 0 else 0
            context_overview["token_summary"]["percentage_breakdown"][component] = round(percentage, 1)

        # === DISPLAY OUTPUT ===
        if display:
            await self._display_context_overview(context_overview)

        return context_overview

    except Exception as e:
        eprint(f"Error generating context overview: {e}")
        return {
            "error": str(e),
            "timestamp": datetime.now().isoformat(),
            "session_id": session_id
        }
get_context_statistics()

Get comprehensive context management statistics

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8948
8949
8950
8951
8952
8953
8954
8955
8956
8957
8958
8959
8960
8961
8962
8963
8964
8965
8966
8967
8968
8969
8970
8971
8972
8973
8974
8975
8976
8977
8978
8979
8980
8981
8982
8983
8984
8985
8986
8987
def get_context_statistics(self) -> dict[str, Any]:
    """Get comprehensive context management statistics"""
    stats = {
        "context_system": "advanced_session_aware",
        "compression_threshold": 0.76,
        "max_tokens": getattr(self, 'max_input_tokens', 8000),
        "session_managers": {},
        "context_usage": {},
        "compression_stats": {}
    }

    # Session manager statistics
    session_managers = self.shared.get("session_managers", {})
    for name, manager in session_managers.items():
        stats["session_managers"][name] = {
            "history_length": len(manager.history if hasattr(manager, 'history') else (manager.get("history", []) if hasattr(manager, 'get') else [])),
            "max_length": manager.max_length if hasattr(manager, 'max_length') else manager.get("max_length", 0),
            "space_name": manager.space_name if hasattr(manager, 'space_name') else manager.get("space_name", "")
        }

    # Context node statistics if available
    if hasattr(self.task_flow, 'context_manager'):
        context_manager = self.task_flow.context_manager
        stats["compression_stats"] = {
            "compression_threshold": context_manager.compression_threshold,
            "max_tokens": context_manager.max_tokens,
            "active_sessions": len(context_manager.session_managers)
        }

    # LLM call statistics from enhanced node
    llm_stats = self.shared.get("llm_call_stats", {})
    if llm_stats:
        stats["context_usage"] = {
            "total_llm_calls": llm_stats.get("total_calls", 0),
            "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
            "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                    1)
        }

    return stats
get_format_quality_report()

Erhalte detaillierten Format-Qualitätsbericht

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8716
8717
8718
8719
8720
8721
8722
8723
8724
8725
8726
8727
8728
8729
8730
8731
8732
8733
8734
def get_format_quality_report(self) -> dict[str, Any]:
    """Erhalte detaillierten Format-Qualitätsbericht"""
    quality_assessment = self.shared.get("quality_assessment", {})

    if not quality_assessment:
        return {"status": "no_assessment", "message": "No recent quality assessment available"}

    quality_details = quality_assessment.get("quality_details", {})

    return {
        "overall_score": quality_details.get("total_score", 0.0),
        "format_adherence": quality_details.get("format_adherence", 0.0),
        "length_adherence": quality_details.get("length_adherence", 0.0),
        "content_quality": quality_details.get("base_quality", 0.0),
        "llm_assessment": quality_details.get("llm_assessment", 0.0),
        "suggestions": quality_assessment.get("suggestions", []),
        "assessment": quality_assessment.get("quality_assessment", "unknown"),
        "format_config_active": quality_details.get("format_config_used", False)
    }
get_session_storage_stats()

Get comprehensive session storage statistics

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9787
9788
9789
9790
9791
9792
9793
9794
9795
9796
9797
9798
9799
9800
9801
9802
9803
9804
9805
9806
9807
9808
9809
9810
9811
9812
9813
9814
9815
9816
9817
9818
9819
9820
9821
9822
9823
9824
9825
9826
9827
9828
9829
9830
9831
9832
9833
9834
9835
9836
9837
9838
9839
9840
9841
9842
9843
9844
9845
9846
9847
9848
9849
9850
9851
9852
9853
9854
def get_session_storage_stats(self) -> dict[str, Any]:
    """Get comprehensive session storage statistics"""
    try:
        stats = {
            "context_manager_active": bool(self.context_manager),
            "total_sessions": 0,
            "session_details": {},
            "storage_summary": {
                "total_messages": 0,
                "context_snapshots": 0,
                "context_entries": 0,
                "regular_messages": 0
            }
        }

        if not self.context_manager:
            return stats

        stats["total_sessions"] = len(self.context_manager.session_managers)

        for session_id, session in self.context_manager.session_managers.items():
            session_stats = {
                "session_type": "chatsession" if hasattr(session, 'history') else "fallback",
                "message_count": 0,
                "context_snapshots": 0,
                "context_entries": 0,
                "regular_messages": 0,
                "storage_size_estimate": 0
            }

            if hasattr(session, 'history'):
                session_stats["message_count"] = len(session.history)

                for message in session.history:
                    content_size = len(str(message))
                    session_stats["storage_size_estimate"] += content_size

                    metadata = message.get("metadata", {})
                    if metadata.get("is_context_snapshot"):
                        session_stats["context_snapshots"] += 1
                    elif metadata.get("is_context_entry"):
                        session_stats["context_entries"] += 1
                    else:
                        session_stats["regular_messages"] += 1

            elif isinstance(session, dict) and 'history' in session:
                session_stats["message_count"] = len(session['history'])
                session_stats["regular_messages"] = len(session['history'])
                session_stats["storage_size_estimate"] = sum(len(str(msg)) for msg in session['history'])

            stats["session_details"][session_id] = session_stats

            # Update totals
            stats["storage_summary"]["total_messages"] += session_stats["message_count"]
            stats["storage_summary"]["context_snapshots"] += session_stats["context_snapshots"]
            stats["storage_summary"]["context_entries"] += session_stats["context_entries"]
            stats["storage_summary"]["regular_messages"] += session_stats["regular_messages"]

        # Estimate total storage size
        stats["storage_summary"]["estimated_total_size_kb"] = sum(
            details["storage_size_estimate"] for details in stats["session_details"].values()
        ) / 1024

        return stats

    except Exception as e:
        eprint(f"Failed to get session storage stats: {e}")
        return {"error": str(e)}
get_task_execution_summary() async

Erhalte detaillierte Zusammenfassung der Task-Ausführung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9115
9116
9117
9118
9119
9120
9121
9122
9123
9124
9125
9126
9127
9128
9129
9130
9131
9132
9133
9134
9135
9136
9137
9138
9139
9140
9141
9142
9143
9144
9145
9146
9147
9148
9149
9150
9151
9152
9153
9154
9155
async def get_task_execution_summary(self) -> dict[str, Any]:
    """Erhalte detaillierte Zusammenfassung der Task-Ausführung"""
    tasks = self.shared.get("tasks", {})
    results_store = self.shared.get("results", {})

    summary = {
        "total_tasks": len(tasks),
        "completed_tasks": [],
        "failed_tasks": [],
        "task_types_used": {},
        "tools_used": [],
        "adaptations": self.shared.get("plan_adaptations", 0),
        "execution_timeline": []
    }

    for task_id, task in tasks.items():
        task_info = {
            "id": task_id,
            "type": task.type,
            "description": task.description,
            "status": task.status,
            "duration": None
        }

        if task.started_at and task.completed_at:
            duration = (task.completed_at - task.started_at).total_seconds()
            task_info["duration"] = duration

        if task.status == "completed":
            summary["completed_tasks"].append(task_info)
            if isinstance(task, ToolTask):
                summary["tools_used"].append(task.tool_name)
        elif task.status == "failed":
            task_info["error"] = task.error
            summary["failed_tasks"].append(task_info)

        # Task types counting
        task_type = task.type
        summary["task_types_used"][task_type] = summary["task_types_used"].get(task_type, 0) + 1

    return summary
get_tool_by_name(tool_name)

Get tool function by name

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10246
10247
10248
def get_tool_by_name(self, tool_name: str) -> Callable | None:
    """Get tool function by name"""
    return self._tool_registry.get(tool_name, {}).get("function")
get_variable(path, default=None)

Get variable using unified system

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8794
8795
8796
def get_variable(self, path: str, default=None):
    """Get variable using unified system"""
    return self.variable_manager.get(path, default)
get_variable_documentation()

Get comprehensive variable system documentation

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8736
8737
8738
8739
8740
8741
8742
8743
8744
8745
8746
8747
8748
8749
8750
8751
8752
8753
8754
8755
8756
8757
8758
8759
8760
8761
8762
8763
8764
8765
8766
def get_variable_documentation(self) -> str:
    """Get comprehensive variable system documentation"""
    docs = []
    docs.append("# Variable System Documentation\n")

    # Available scopes
    docs.append("## Available Scopes:")
    scope_info = self.variable_manager.get_scope_info()
    for scope_name, info in scope_info.items():
        docs.append(f"- `{scope_name}`: {info['type']} with {info.get('keys', 'N/A')} keys")

    docs.append("\n## Syntax Options:")
    docs.append("- `{{ variable.path }}` - Full path resolution")
    docs.append("- `{variable}` - Simple variable (no dots)")
    docs.append("- `$variable` - Shell-style variable")

    docs.append("\n## Example Usage:")
    docs.append("- `{{ results.task_1.data }}` - Get result from task_1")
    docs.append("- `{{ user.name }}` - Get user name")
    docs.append("- `{agent_name}` - Simple agent name")
    docs.append("- `$timestamp` - System timestamp")

    # Available variables
    docs.append("\n## Available Variables:")
    variables = self.variable_manager.get_available_variables()
    for scope_name, scope_vars in variables.items():
        docs.append(f"\n### {scope_name}:")
        for _var_name, var_info in scope_vars.items():
            docs.append(f"- `{var_info['path']}`: {var_info['preview']} ({var_info['type']})")

    return "\n".join(docs)
initialize_context_awareness() async

Enhanced context awareness with session management

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8828
8829
8830
8831
8832
8833
8834
8835
8836
8837
8838
8839
8840
8841
8842
8843
8844
8845
8846
8847
8848
8849
8850
8851
8852
8853
8854
8855
8856
8857
8858
8859
8860
8861
8862
8863
8864
8865
8866
8867
8868
8869
8870
8871
8872
8873
8874
8875
8876
8877
8878
8879
8880
8881
8882
8883
8884
8885
8886
8887
8888
8889
8890
8891
8892
8893
8894
8895
8896
8897
8898
8899
8900
8901
8902
8903
8904
8905
8906
8907
8908
8909
8910
8911
8912
8913
8914
async def initialize_context_awareness(self):
    """Enhanced context awareness with session management"""

    # Initialize session if not already done
    session_id = self.shared.get("session_id", self.active_session)
    if not self.shared.get("session_initialized"):
        await self.initialize_session_context(session_id)

    # Ensure tool capabilities are loaded
    # add tqdm prigress bar

    from tqdm import tqdm

    if hasattr(self.task_flow, 'llm_reasoner'):
        if "read_from_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_read_from_variables'):
            await self.add_tool(lambda scope, key, purpose: self.task_flow.llm_reasoner._execute_read_from_variables({"scope": scope, "key": key, "purpose": purpose}), "read_from_variables", "Read from variables")
        if "write_to_variables" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_write_to_variables'):
            await self.add_tool(lambda scope, key, value, description: self.task_flow.llm_reasoner._execute_write_to_variables({"scope": scope, "key": key, "value": value, "description": description}), "write_to_variables", "Write to variables")

        if "internal_reasoning" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_internal_reasoning'):
            async def internal_reasoning_tool(thought:str, thought_number:int, total_thoughts:int, next_thought_needed:bool, current_focus:str, key_insights:list[str], potential_issues:list[str], confidence_level:float):
                args = {
                    "thought": thought,
                    "thought_number": thought_number,
                    "total_thoughts": total_thoughts,
                    "next_thought_needed": next_thought_needed,
                    "current_focus": current_focus,
                    "key_insights": key_insights,
                    "potential_issues": potential_issues,
                    "confidence_level": confidence_level
                }
                return await self.task_flow.llm_reasoner._execute_internal_reasoning(args, self.shared)
            await self.add_tool(internal_reasoning_tool, "internal_reasoning", "Internal reasoning")

        if "manage_internal_task_stack" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_manage_task_stack'):
            async def manage_internal_task_stack_tool(action:str, task_description:str, outline_step_ref:str):
                args = {
                    "action": action,
                    "task_description": task_description,
                    "outline_step_ref": outline_step_ref
                }
                return await self.task_flow.llm_reasoner._execute_manage_task_stack(args, self.shared)
            await self.add_tool(manage_internal_task_stack_tool, "manage_internal_task_stack", "Manage internal task stack")

        if "outline_step_completion" not in self.shared["available_tools"] and hasattr(self.task_flow.llm_reasoner, '_execute_outline_step_completion'):
            async def outline_step_completion_tool(step_completed:bool, completion_evidence:str, next_step_focus:str):
                args = {
                    "step_completed": step_completed,
                    "completion_evidence": completion_evidence,
                    "next_step_focus": next_step_focus
                }
                return await self.task_flow.llm_reasoner._execute_outline_step_completion(args, self.shared)
            await self.add_tool(outline_step_completion_tool, "outline_step_completion", "Outline step completion")


    registered_tools = set(self._tool_registry.keys())
    cached_capabilities = list(self._tool_capabilities.keys())  # Create a copy of
    for tool_name in cached_capabilities:
        if tool_name in self._tool_capabilities and tool_name not in registered_tools:
            del self._tool_capabilities[tool_name]
            iprint(f"Removed outdated capability for unavailable tool: {tool_name}")

    for tool_name in tqdm(self.shared["available_tools"], desc=f"Agent {self.amd.name} Analyzing Tools", unit="tool", colour="green", total=len(self.shared["available_tools"])):
        if tool_name not in self._tool_capabilities:
            tool_info = self._tool_registry.get(tool_name, {})
            description = tool_info.get("description", "No description")
            with Spinner(f"Analyzing tool {tool_name}"):
                await self._analyze_tool_capabilities(tool_name, description, tool_info.get("args_schema", "()"))

        if tool_name in self._tool_capabilities:
            function = self._tool_registry[tool_name]["function"]
            if not isinstance(self._tool_capabilities[tool_name], dict):
                self._tool_capabilities[tool_name] = {}
            self._tool_capabilities[tool_name]["args_schema"] = get_args_schema(function)

    # Set enhanced system context
    self.shared["system_context"] = {
        "capabilities_summary": self._build_capabilities_summary(),
        "tool_count": len(self.shared["available_tools"]),
        "analysis_loaded": len(self._tool_capabilities),
        "intelligence_level": "high" if self._tool_capabilities else "basic",
        "context_management": "advanced_session_aware",
        "session_managers": len(self.shared.get("session_managers", {})),
    }


    rprint("Advanced context awareness initialized with session management")
initialize_session_context(session_id='default', max_history=200) async

Vereinfachte Session-Initialisierung über UnifiedContextManager

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8802
8803
8804
8805
8806
8807
8808
8809
8810
8811
8812
8813
8814
8815
8816
8817
8818
8819
8820
8821
8822
8823
8824
8825
8826
async def initialize_session_context(self, session_id: str = "default", max_history: int = 200) -> bool:
    """Vereinfachte Session-Initialisierung über UnifiedContextManager"""
    try:
        # Delegation an UnifiedContextManager
        session = await self.context_manager.initialize_session(session_id, max_history)

        # Ensure Variable Manager integration
        if not self.context_manager.variable_manager:
            self.context_manager.variable_manager = self.variable_manager

        # Update shared state (minimal - primary data now in context_manager)
        self.shared["active_session_id"] = session_id
        self.shared["session_initialized"] = True

        # Legacy support: Keep session_managers reference in shared for backward compatibility
        self.shared["session_managers"] = self.context_manager.session_managers

        rprint(f"Session context initialized for {session_id} via UnifiedContextManager")
        return True

    except Exception as e:
        eprint(f"Session context initialization failed: {e}")
        import traceback
        print(traceback.format_exc())
        return False
list_available_checkpoints(max_age_hours=168)

List all available checkpoints with metadata

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9856
9857
9858
9859
9860
9861
9862
9863
9864
9865
9866
9867
9868
9869
9870
9871
9872
9873
9874
9875
9876
9877
9878
9879
9880
9881
9882
9883
9884
9885
9886
9887
9888
9889
9890
9891
9892
9893
9894
9895
9896
9897
9898
9899
9900
9901
9902
9903
9904
9905
9906
9907
9908
9909
9910
9911
9912
9913
9914
9915
9916
9917
9918
9919
9920
9921
9922
9923
9924
9925
9926
9927
9928
9929
def list_available_checkpoints(self, max_age_hours: int = 168) -> list[dict[str, Any]]:  # Default 1 week
    """List all available checkpoints with metadata"""
    try:
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

        if not os.path.exists(folder):
            return []

        checkpoints = []
        for file in os.listdir(folder):
            if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                filepath = os.path.join(folder, file)
                try:
                    # Get file info
                    file_stat = os.stat(filepath)
                    file_size = file_stat.st_size
                    modified_time = datetime.fromtimestamp(file_stat.st_mtime)

                    # Extract timestamp from filename
                    timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                    if timestamp_str == 'final_checkpoint':
                        checkpoint_time = modified_time
                        checkpoint_type = "final"
                    else:
                        checkpoint_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")
                        checkpoint_type = "regular"

                    # Check age
                    age_hours = (datetime.now() - checkpoint_time).total_seconds() / 3600
                    if age_hours <= max_age_hours:

                        # Try to load checkpoint metadata without full loading
                        metadata = {}
                        try:
                            with open(filepath, 'rb') as f:
                                checkpoint = pickle.load(f)
                            metadata = {
                                "tasks_count": len(checkpoint.task_state) if checkpoint.task_state else 0,
                                "world_model_entries": len(checkpoint.world_model) if checkpoint.world_model else 0,
                                "session_id": checkpoint.metadata.get("session_id", "unknown") if hasattr(
                                    checkpoint, 'metadata') and checkpoint.metadata else "unknown",
                                "last_query": checkpoint.metadata.get("last_query", "unknown")[:100] if hasattr(
                                    checkpoint, 'metadata') and checkpoint.metadata else "unknown"
                            }
                        except:
                            metadata = {"load_error": True}

                        checkpoints.append({
                            "filepath": filepath,
                            "filename": file,
                            "checkpoint_type": checkpoint_type,
                            "timestamp": checkpoint_time.isoformat(),
                            "age_hours": round(age_hours, 1),
                            "file_size_kb": round(file_size / 1024, 1),
                            "metadata": metadata
                        })

                except Exception as e:
                    import traceback
                    print(traceback.format_exc())
                    wprint(f"Could not analyze checkpoint file {file}: {e}")
                    continue

        # Sort by timestamp (newest first)
        checkpoints.sort(key=lambda x: x["timestamp"], reverse=True)

        return checkpoints

    except Exception as e:
        import traceback
        print(traceback.format_exc())
        eprint(f"Failed to list checkpoints: {e}")
        return []
load_context_from_session(session_id, context_type='full') async

Load context from ChatSession storage

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9667
9668
9669
9670
9671
9672
9673
9674
9675
9676
9677
9678
9679
9680
9681
9682
9683
9684
9685
9686
9687
9688
9689
9690
9691
9692
9693
9694
9695
9696
9697
9698
9699
9700
9701
9702
9703
9704
9705
9706
9707
9708
9709
9710
async def load_context_from_session(self, session_id: str, context_type: str = "full") -> dict[str, Any]:
    """Load context from ChatSession storage"""
    try:
        if not self.context_manager:
            return {"error": "Context manager not available"}

        session = self.context_manager.session_managers.get(session_id)
        if not session:
            return {"error": f"Session {session_id} not found"}

        # Search for context snapshots in session history
        context_snapshots = []

        if hasattr(session, 'history'):
            for message in reversed(session.history):  # Search from newest
                if (message.get("role") == "system" and
                    message.get("metadata", {}).get("is_context_snapshot") and
                    message.get("metadata", {}).get("context_type") == context_type):

                    try:
                        # Extract context data
                        content = message.get("content", "")
                        if content.startswith(f"[CONTEXT_SNAPSHOT_{context_type.upper()}]"):
                            json_data = content.replace(f"[CONTEXT_SNAPSHOT_{context_type.upper()}] ", "")
                            context_data = json.loads(json_data)
                            context_snapshots.append({
                                "context": context_data,
                                "timestamp": message.get("timestamp"),
                                "metadata": message.get("metadata", {})
                            })
                    except Exception as e:
                        wprint(f"Failed to parse context snapshot: {e}")

        if context_snapshots:
            # Return most recent context snapshot
            latest_context = context_snapshots[0]
            rprint(f"Loaded context snapshot from session {session_id} (timestamp: {latest_context['timestamp']})")
            return latest_context["context"]
        else:
            return {"error": f"No context snapshots of type '{context_type}' found in session {session_id}"}

    except Exception as e:
        eprint(f"Failed to load context from session: {e}")
        return {"error": str(e)}
load_latest_checkpoint(auto_restore_history=True, max_age_hours=24) async

Vereinfachtes Checkpoint-Laden mit automatischer History-Wiederherstellung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9387
9388
9389
9390
9391
9392
9393
9394
9395
9396
9397
9398
9399
9400
9401
9402
9403
9404
9405
9406
9407
9408
9409
9410
9411
9412
9413
9414
9415
9416
9417
9418
9419
9420
9421
9422
9423
9424
9425
9426
9427
9428
9429
9430
9431
9432
9433
9434
9435
9436
9437
9438
9439
9440
9441
9442
9443
9444
9445
9446
async def load_latest_checkpoint(self, auto_restore_history: bool = True, max_age_hours: int = 24) -> dict[
    str, Any]:
    """Vereinfachtes Checkpoint-Laden mit automatischer History-Wiederherstellung"""
    try:
        from toolboxv2 import get_app
        folder = str(get_app().data_dir) + '/Agents/checkpoint/' + self.amd.name

        if not os.path.exists(folder):
            return {"success": False, "error": "Kein Checkpoint-Verzeichnis gefunden"}

        # Finde neuesten Checkpoint
        checkpoint_files = []
        for file in os.listdir(folder):
            if file.endswith('.pkl') and file.startswith('agent_checkpoint_'):
                filepath = os.path.join(folder, file)
                try:
                    timestamp_str = file.replace('agent_checkpoint_', '').replace('.pkl', '')
                    if timestamp_str == 'final_checkpoint':
                        file_time = datetime.fromtimestamp(os.path.getmtime(filepath))
                    else:
                        file_time = datetime.strptime(timestamp_str, "%Y%m%d_%H%M%S")

                    age_hours = (datetime.now() - file_time).total_seconds() / 3600
                    if age_hours <= max_age_hours:
                        checkpoint_files.append((filepath, file_time, age_hours))
                except Exception:
                    continue

        if not checkpoint_files:
            return {"success": False, "error": f"Keine gültigen Checkpoints in {max_age_hours} Stunden gefunden"}

        # Lade neuesten Checkpoint
        checkpoint_files.sort(key=lambda x: x[1], reverse=True)
        latest_checkpoint_path, latest_timestamp, age_hours = checkpoint_files[0]

        rprint(f"Lade Checkpoint: {latest_checkpoint_path} (Alter: {age_hours:.1f}h)")

        with open(latest_checkpoint_path, 'rb') as f:
            checkpoint: AgentCheckpoint = pickle.load(f)

        # Stelle Agent-Status wieder her
        restore_stats = await self._restore_from_checkpoint_simplified(checkpoint, auto_restore_history)

        # Re-initialisiere Kontext-Awareness
        await self.initialize_context_awareness()

        return {
            "success": True,
            "checkpoint_file": latest_checkpoint_path,
            "checkpoint_age_hours": age_hours,
            "checkpoint_timestamp": latest_timestamp.isoformat(),
            "available_checkpoints": len(checkpoint_files),
            "restore_stats": restore_stats
        }

    except Exception as e:
        eprint(f"Checkpoint-Laden fehlgeschlagen: {e}")
        import traceback
        print(traceback.format_exc())
        return {"success": False, "error": str(e)}
pause() async

Pause agent execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9212
9213
9214
9215
9216
9217
9218
9219
9220
9221
9222
9223
9224
9225
async def pause(self) -> bool:
    """Pause agent execution"""
    if not self.is_running:
        return False

    self.is_paused = True
    self.shared["system_status"] = "paused"

    # Create checkpoint
    checkpoint = await self._create_checkpoint()
    await self._save_checkpoint(checkpoint)

    rprint("Agent execution paused")
    return True
resume() async

Resume agent execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9227
9228
9229
9230
9231
9232
9233
9234
9235
9236
async def resume(self) -> bool:
    """Resume agent execution"""
    if not self.is_paused:
        return False

    self.is_paused = False
    self.shared["system_status"] = "running"

    rprint("Agent execution resumed")
    return True
save_context_to_file(session_id=None) async

Save current context to file

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10216
10217
10218
10219
10220
10221
10222
10223
10224
10225
10226
10227
10228
10229
10230
10231
async def save_context_to_file(self, session_id: str = None) -> bool:
    """Save current context to file"""
    try:
        context = await self.get_context(session_id=session_id, format_for_llm=False)

        filepath = self._get_context_path(session_id)

        with open(filepath, 'w', encoding='utf-8') as f:
            json.dump(context, f, indent=2, ensure_ascii=False, default=str)

        rprint(f"Context saved to: {filepath}")
        return True

    except Exception as e:
        eprint(f"Failed to save context: {e}")
        return False
save_context_to_session(session_id=None, context_type='full') async

Save current context to ChatSession for persistent storage

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
9624
9625
9626
9627
9628
9629
9630
9631
9632
9633
9634
9635
9636
9637
9638
9639
9640
9641
9642
9643
9644
9645
9646
9647
9648
9649
9650
9651
9652
9653
9654
9655
9656
9657
9658
9659
9660
9661
9662
9663
9664
9665
async def save_context_to_session(self, session_id: str = None, context_type: str = "full") -> bool:
    """Save current context to ChatSession for persistent storage"""
    try:
        session_id = session_id or self.shared.get("session_id", "default")

        if not self.context_manager:
            eprint("Context manager not available")
            return False

        # Build comprehensive context
        unified_context = await self.context_manager.build_unified_context(session_id, None, context_type)

        # Create context message for session storage
        context_message = {
            "role": "system",
            "content": f"[CONTEXT_SNAPSHOT_{context_type.upper()}] " + json.dumps(unified_context, default=str),
            "timestamp": datetime.now().isoformat(),
            "context_type": context_type,
            "metadata": {
                "is_context_snapshot": True,
                "context_version": "2.0",
                "agent_name": self.amd.name,
                "session_stats": unified_context.get("session_stats", {}),
                "variables_count": len(unified_context.get("variables", {}).get("recent_results", [])),
                "execution_state": unified_context.get("execution_state", {}).get("system_status", "unknown")
            }
        }

        # Store in session
        await self.context_manager.add_interaction(
            session_id,
            "system",
            context_message["content"],
            metadata=context_message["metadata"]
        )

        rprint(f"Context snapshot saved to session {session_id} (type: {context_type})")
        return True

    except Exception as e:
        eprint(f"Failed to save context to session: {e}")
        return False
set_persona(name, style='professional', tone='friendly', personality_traits=None, apply_method='system_prompt', integration_level='light', custom_instructions='')

Set agent persona mit erweiterten Konfigurationsmöglichkeiten

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8989
8990
8991
8992
8993
8994
8995
8996
8997
8998
8999
9000
9001
9002
9003
9004
9005
9006
def set_persona(self, name: str, style: str = "professional", tone: str = "friendly",
                personality_traits: list[str] = None, apply_method: str = "system_prompt",
                integration_level: str = "light", custom_instructions: str = ""):
    """Set agent persona mit erweiterten Konfigurationsmöglichkeiten"""
    if personality_traits is None:
        personality_traits = ["helpful", "concise"]

    self.amd.persona = PersonaConfig(
        name=name,
        style=style,
        tone=tone,
        personality_traits=personality_traits,
        custom_instructions=custom_instructions,
        apply_method=apply_method,
        integration_level=integration_level
    )

    rprint(f"Persona set: {name} ({style}, {tone}) - Method: {apply_method}, Level: {integration_level}")
set_response_format(response_format, text_length, custom_instructions='', quality_threshold=0.7)

Dynamische Format- und Längen-Konfiguration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8590
8591
8592
8593
8594
8595
8596
8597
8598
8599
8600
8601
8602
8603
8604
8605
8606
8607
8608
8609
8610
8611
8612
8613
8614
8615
8616
8617
8618
8619
8620
8621
8622
8623
8624
8625
8626
8627
8628
8629
8630
8631
8632
8633
def set_response_format(
    self,
    response_format: str,
    text_length: str,
    custom_instructions: str = "",
    quality_threshold: float = 0.7
):
    """Dynamische Format- und Längen-Konfiguration"""

    # Validiere Eingaben
    try:
        ResponseFormat(response_format)
        TextLength(text_length)
    except ValueError:
        available_formats = [f.value for f in ResponseFormat]
        available_lengths = [l.value for l in TextLength]
        raise ValueError(
            f"Invalid format or length. "
            f"Available formats: {available_formats}. "
            f"Available lengths: {available_lengths}"
        )

    # Erstelle oder aktualisiere Persona
    if not self.amd.persona:
        self.amd.persona = PersonaConfig(name="Assistant")

    # Erstelle Format-Konfiguration
    format_config = FormatConfig(
        response_format=ResponseFormat(response_format),
        text_length=TextLength(text_length),
        custom_instructions=custom_instructions,
        quality_threshold=quality_threshold
    )

    self.amd.persona.format_config = format_config

    # Aktualisiere Personality Traits mit Format-Hinweisen
    self._update_persona_with_format(response_format, text_length)

    # Update shared state
    self.shared["persona_config"] = self.amd.persona
    self.shared["format_config"] = format_config

    rprint(f"Response format set: {response_format}, length: {text_length}")
set_variable(path, value)

Set variable using unified system

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8790
8791
8792
def set_variable(self, path: str, value: Any):
    """Set variable using unified system"""
    self.variable_manager.set(path, value)
setup_a2a_server(host='0.0.0.0', port=5000, **kwargs)

Setup A2A server for bidirectional communication

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10495
10496
10497
10498
10499
10500
10501
10502
10503
10504
10505
10506
10507
10508
10509
10510
10511
10512
10513
10514
10515
10516
10517
10518
10519
10520
10521
10522
10523
10524
10525
def setup_a2a_server(self, host: str = "0.0.0.0", port: int = 5000, **kwargs):
    """Setup A2A server for bidirectional communication"""
    if not A2A_AVAILABLE:
        wprint("A2A not available, cannot setup server")
        return

    try:
        self.a2a_server = A2AServer(
            host=host,
            port=port,
            agent_card=AgentCard(
                name=self.amd.name,
                description="Production-ready PocketFlow agent",
                version="1.0.0"
            ),
            **kwargs
        )

        # Register agent methods
        @self.a2a_server.route("/run")
        async def handle_run(request_data):
            query = request_data.get("query", "")
            session_id = request_data.get("session_id", "a2a_session")

            response = await self.a_run(query, session_id=session_id)
            return {"response": response}

        rprint(f"A2A server setup on {host}:{port}")

    except Exception as e:
        eprint(f"Failed to setup A2A server: {e}")
setup_mcp_server(host='0.0.0.0', port=8000, name=None, **kwargs)

Setup MCP server

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10527
10528
10529
10530
10531
10532
10533
10534
10535
10536
10537
10538
10539
10540
10541
10542
10543
10544
10545
10546
def setup_mcp_server(self, host: str = "0.0.0.0", port: int = 8000, name: str = None, **kwargs):
    """Setup MCP server"""
    if not MCP_AVAILABLE:
        wprint("MCP not available, cannot setup server")
        return

    try:
        server_name = name or f"{self.amd.name}_MCP"
        self.mcp_server = FastMCP(server_name)

        # Register agent as MCP tool
        @self.mcp_server.tool()
        async def agent_run(query: str, session_id: str = "mcp_session") -> str:
            """Execute agent with given query"""
            return await self.a_run(query, session_id=session_id)

        rprint(f"MCP server setup: {server_name}")

    except Exception as e:
        eprint(f"Failed to setup MCP server: {e}")
start_servers() async

Start all configured servers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
10550
10551
10552
10553
10554
10555
10556
10557
10558
10559
10560
10561
10562
async def start_servers(self):
    """Start all configured servers"""
    tasks = []

    if self.a2a_server:
        tasks.append(asyncio.create_task(self.a2a_server.start()))

    if self.mcp_server:
        tasks.append(asyncio.create_task(self.mcp_server.run()))

    if tasks:
        rprint(f"Starting {len(tasks)} servers...")
        await asyncio.gather(*tasks, return_exceptions=True)
status(pretty_print=False) async

Get comprehensive agent status with optional pretty printing

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11138
11139
11140
11141
11142
11143
11144
11145
11146
11147
11148
11149
11150
11151
11152
11153
11154
11155
11156
11157
11158
11159
11160
11161
11162
11163
11164
11165
11166
11167
11168
11169
11170
11171
11172
11173
11174
11175
11176
11177
11178
11179
11180
11181
11182
11183
11184
11185
11186
11187
11188
11189
11190
11191
11192
11193
11194
11195
11196
11197
11198
11199
11200
11201
11202
11203
11204
11205
11206
11207
11208
11209
11210
11211
11212
11213
11214
11215
11216
11217
11218
11219
11220
11221
11222
11223
11224
11225
11226
11227
11228
11229
11230
11231
11232
11233
11234
11235
11236
11237
11238
11239
11240
11241
11242
11243
11244
11245
11246
11247
11248
11249
11250
11251
11252
11253
11254
11255
11256
11257
11258
11259
11260
11261
11262
11263
11264
11265
11266
11267
11268
11269
11270
11271
11272
11273
11274
11275
11276
11277
11278
11279
11280
11281
11282
11283
11284
11285
11286
11287
11288
11289
11290
11291
11292
11293
11294
11295
11296
11297
11298
11299
11300
11301
11302
11303
11304
11305
11306
11307
11308
11309
11310
11311
11312
11313
11314
11315
11316
11317
11318
11319
11320
11321
11322
11323
11324
11325
11326
11327
11328
11329
11330
11331
11332
11333
11334
11335
11336
11337
11338
11339
11340
11341
11342
11343
11344
11345
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356
11357
11358
11359
11360
11361
11362
11363
11364
11365
11366
11367
11368
11369
11370
11371
11372
11373
11374
11375
11376
11377
11378
11379
11380
11381
11382
11383
11384
11385
11386
11387
11388
11389
11390
11391
11392
11393
11394
11395
11396
11397
11398
11399
11400
11401
11402
11403
11404
11405
11406
11407
11408
11409
11410
11411
11412
11413
11414
11415
11416
11417
11418
11419
11420
11421
11422
11423
11424
11425
11426
11427
11428
11429
11430
11431
11432
11433
11434
11435
11436
11437
11438
11439
11440
11441
11442
11443
async def status(self, pretty_print: bool = False) -> dict[str, Any] | str:
    """Get comprehensive agent status with optional pretty printing"""

    # Core status information
    base_status = {
        "agent_info": {
            "name": self.amd.name,
            "version": "2.0",
            "type": "FlowAgent"
        },
        "runtime_status": {
            "status": self.shared.get("system_status", "idle"),
            "is_running": self.is_running,
            "is_paused": self.is_paused,
            "uptime_seconds": (datetime.now() - getattr(self, '_start_time', datetime.now())).total_seconds()
        },
        "task_execution": {
            "total_tasks": len(self.shared.get("tasks", {})),
            "active_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "running"]),
            "completed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "completed"]),
            "failed_tasks": len([t for t in self.shared.get("tasks", {}).values() if t.status == "failed"]),
            "plan_adaptations": self.shared.get("plan_adaptations", 0)
        },
        "conversation": {
            "turns": len(self.shared.get("conversation_history", [])),
            "session_id": self.shared.get("session_id", self.active_session),
            "current_user": self.shared.get("user_id"),
            "last_query": self.shared.get("current_query", "")[:100] + "..." if len(
                self.shared.get("current_query", "")) > 100 else self.shared.get("current_query", "")
        },
        "capabilities": {
            "available_tools": len(self.shared.get("available_tools", [])),
            "tool_names": list(self.shared.get("available_tools", [])),
            "analyzed_tools": len(self._tool_capabilities),
            "world_model_size": len(self.shared.get("world_model", {})),
            "intelligence_level": "high" if self._tool_capabilities else "basic"
        },
        "memory_context": {
            "session_initialized": self.shared.get("session_initialized", False),
            "session_managers": len(self.shared.get("session_managers", {})),
            "context_system": "advanced_session_aware" if self.shared.get("session_initialized") else "basic",
            "variable_scopes": len(self.variable_manager.get_scope_info()) if hasattr(self,
                                                                                      'variable_manager') else 0
        },
        "performance": {
            "total_cost": self.total_cost,
            "checkpoint_enabled": self.enable_pause_resume,
            "last_checkpoint": self.last_checkpoint.isoformat() if self.last_checkpoint else None,
            "max_parallel_tasks": self.max_parallel_tasks
        },
        "servers": {
            "a2a_server": self.a2a_server is not None,
            "mcp_server": self.mcp_server is not None,
            "server_count": sum([self.a2a_server is not None, self.mcp_server is not None])
        },
        "configuration": {
            "fast_llm_model": self.amd.fast_llm_model,
            "complex_llm_model": self.amd.complex_llm_model,
            "use_fast_response": getattr(self.amd, 'use_fast_response', False),
            "max_input_tokens": getattr(self.amd, 'max_input_tokens', 8000),
            "persona_configured": self.amd.persona is not None,
            "format_config": bool(getattr(self.amd.persona, 'format_config', None)) if self.amd.persona else False
        }
    }

    # Add detailed execution summary if tasks exist
    tasks = self.shared.get("tasks", {})
    if tasks:
        task_types_used = {}
        tools_used = []
        execution_timeline = []

        for task_id, task in tasks.items():
            # Count task types
            task_type = getattr(task, 'type', 'unknown')
            task_types_used[task_type] = task_types_used.get(task_type, 0) + 1

            # Collect tools used
            if hasattr(task, 'tool_name') and task.tool_name:
                tools_used.append(task.tool_name)

            # Timeline info
            if hasattr(task, 'started_at') and task.started_at:
                timeline_entry = {
                    "task_id": task_id,
                    "type": task_type,
                    "started": task.started_at.isoformat(),
                    "status": getattr(task, 'status', 'unknown')
                }
                if hasattr(task, 'completed_at') and task.completed_at:
                    timeline_entry["completed"] = task.completed_at.isoformat()
                    timeline_entry["duration"] = (task.completed_at - task.started_at).total_seconds()
                execution_timeline.append(timeline_entry)

        base_status["task_execution"].update({
            "task_types_used": task_types_used,
            "tools_used": list(set(tools_used)),
            "execution_timeline": execution_timeline[-5:]  # Last 5 tasks
        })

    # Add context statistics
    if hasattr(self.task_flow, 'context_manager'):
        context_manager = self.task_flow.context_manager
        base_status["memory_context"].update({
            "compression_threshold": context_manager.compression_threshold,
            "max_tokens": context_manager.max_tokens,
            "active_context_sessions": len(getattr(context_manager, 'session_managers', {}))
        })

    # Add variable system info
    if hasattr(self, 'variable_manager'):
        available_vars = self.variable_manager.get_available_variables()
        scope_info = self.variable_manager.get_scope_info()

        base_status["variable_system"] = {
            "total_scopes": len(scope_info),
            "scope_names": list(scope_info.keys()),
            "total_variables": sum(len(vars) for vars in available_vars.values()),
            "scope_details": {
                scope: {"type": info["type"], "variables": len(available_vars.get(scope, {}))}
                for scope, info in scope_info.items()
            }
        }

    # Add format quality info if available
    quality_assessment = self.shared.get("quality_assessment", {})
    if quality_assessment:
        quality_details = quality_assessment.get("quality_details", {})
        base_status["format_quality"] = {
            "overall_score": quality_details.get("total_score", 0.0),
            "format_adherence": quality_details.get("format_adherence", 0.0),
            "length_adherence": quality_details.get("length_adherence", 0.0),
            "content_quality": quality_details.get("base_quality", 0.0),
            "assessment": quality_assessment.get("quality_assessment", "unknown"),
            "has_suggestions": bool(quality_assessment.get("suggestions", []))
        }

    # Add LLM usage statistics
    llm_stats = self.shared.get("llm_call_stats", {})
    if llm_stats:
        base_status["llm_usage"] = {
            "total_calls": llm_stats.get("total_calls", 0),
            "context_compression_rate": llm_stats.get("context_compression_rate", 0.0),
            "average_context_tokens": llm_stats.get("context_tokens_used", 0) / max(llm_stats.get("total_calls", 1),
                                                                                    1),
            "total_tokens_used": llm_stats.get("total_tokens_used", 0)
        }

    # Add timestamp
    base_status["timestamp"] = datetime.now().isoformat()

    base_status["context_statistic"] = self.get_context_statistics()
    if not pretty_print:
        base_status["agent_context"] = await self.get_context_overview()
        return base_status

    # Pretty print using EnhancedVerboseOutput
    try:
        from toolboxv2.mods.isaa.extras.verbose_output import EnhancedVerboseOutput
        verbose_output = EnhancedVerboseOutput(verbose=True)

        # Header
        verbose_output.log_header(f"Agent Status: {base_status['agent_info']['name']}")

        # Runtime Status
        status_color = {
            "running": "SUCCESS",
            "paused": "WARNING",
            "idle": "INFO",
            "error": "ERROR"
        }.get(base_status["runtime_status"]["status"], "INFO")

        getattr(verbose_output, f"print_{status_color.lower()}")(
            f"Status: {base_status['runtime_status']['status'].upper()}"
        )

        # Task Execution Summary
        task_exec = base_status["task_execution"]
        if task_exec["total_tasks"] > 0:
            verbose_output.formatter.print_section(
                "Task Execution",
                f"Total: {task_exec['total_tasks']} | "
                f"Completed: {task_exec['completed_tasks']} | "
                f"Failed: {task_exec['failed_tasks']} | "
                f"Active: {task_exec['active_tasks']}\n"
                f"Adaptations: {task_exec['plan_adaptations']}"
            )

            if task_exec.get("tools_used"):
                verbose_output.formatter.print_section(
                    "Tools Used",
                    ", ".join(task_exec["tools_used"])
                )

        # Capabilities
        caps = base_status["capabilities"]
        verbose_output.formatter.print_section(
            "Capabilities",
            f"Intelligence Level: {caps['intelligence_level']}\n"
            f"Available Tools: {caps['available_tools']}\n"
            f"Analyzed Tools: {caps['analyzed_tools']}\n"
            f"World Model Size: {caps['world_model_size']}"
        )

        # Memory & Context
        memory = base_status["memory_context"]
        verbose_output.formatter.print_section(
            "Memory & Context",
            f"Context System: {memory['context_system']}\n"
            f"Session Managers: {memory['session_managers']}\n"
            f"Variable Scopes: {memory['variable_scopes']}\n"
            f"Session Initialized: {memory['session_initialized']}"
        )

        # Context Statistics
        stats = base_status["context_statistic"]
        verbose_output.formatter.print_section(
            "Context & Stats",
            f"Compression Stats: {stats['compression_stats']}\n"
            f"Session Usage: {stats['context_usage']}\n"
            f"Session Managers: {stats['session_managers']}\n"
        )

        # Configuration
        config = base_status["configuration"]
        verbose_output.formatter.print_section(
            "Configuration",
            f"Fast LLM: {config['fast_llm_model']}\n"
            f"Complex LLM: {config['complex_llm_model']}\n"
            f"Max Tokens: {config['max_input_tokens']}\n"
            f"Persona: {'Configured' if config['persona_configured'] else 'Default'}\n"
            f"Format Config: {'Active' if config['format_config'] else 'None'}"
        )

        # Performance
        perf = base_status["performance"]
        verbose_output.formatter.print_section(
            "Performance",
            f"Total Cost: ${perf['total_cost']:.4f}\n"
            f"Checkpointing: {'Enabled' if perf['checkpoint_enabled'] else 'Disabled'}\n"
            f"Max Parallel Tasks: {perf['max_parallel_tasks']}\n"
            f"Last Checkpoint: {perf['last_checkpoint'] or 'None'}"
        )

        # Variable System Details
        if "variable_system" in base_status:
            var_sys = base_status["variable_system"]
            scope_details = []
            for scope, details in var_sys["scope_details"].items():
                scope_details.append(f"{scope}: {details['variables']} variables ({details['type']})")

            verbose_output.formatter.print_section(
                "Variable System",
                f"Total Scopes: {var_sys['total_scopes']}\n"
                f"Total Variables: {var_sys['total_variables']}\n" +
                "\n".join(scope_details)
            )

        # Format Quality
        if "format_quality" in base_status:
            quality = base_status["format_quality"]
            verbose_output.formatter.print_section(
                "Format Quality",
                f"Overall Score: {quality['overall_score']:.2f}\n"
                f"Format Adherence: {quality['format_adherence']:.2f}\n"
                f"Length Adherence: {quality['length_adherence']:.2f}\n"
                f"Content Quality: {quality['content_quality']:.2f}\n"
                f"Assessment: {quality['assessment']}"
            )

        # LLM Usage
        if "llm_usage" in base_status:
            llm = base_status["llm_usage"]
            verbose_output.formatter.print_section(
                "LLM Usage Statistics",
                f"Total Calls: {llm['total_calls']}\n"
                f"Avg Context Tokens: {llm['average_context_tokens']:.1f}\n"
                f"Total Tokens: {llm['total_tokens_used']}\n"
                f"Compression Rate: {llm['context_compression_rate']:.2%}"
            )

        # Servers
        servers = base_status["servers"]
        if servers["server_count"] > 0:
            server_status = []
            if servers["a2a_server"]:
                server_status.append("A2A Server: Active")
            if servers["mcp_server"]:
                server_status.append("MCP Server: Active")

            verbose_output.formatter.print_section(
                "Servers",
                "\n".join(server_status)
            )

        verbose_output.print_separator()
        await self.get_context_overview(display=True)
        verbose_output.print_separator()
        verbose_output.print_info(f"Status generated at: {base_status['timestamp']}")

        return "Status printed above"

    except Exception:
        # Fallback to JSON if pretty print fails
        import json
        return json.dumps(base_status, indent=2, default=str)
unbind(preserve_shared_data=False)

Unbind this agent from any binding configuration.

Parameters:

Name Type Description Default
preserve_shared_data bool

Whether to preserve shared data in the agent after unbinding

False

Returns:

Name Type Description
dict

Unbinding result with statistics

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11592
11593
11594
11595
11596
11597
11598
11599
11600
11601
11602
11603
11604
11605
11606
11607
11608
11609
11610
11611
11612
11613
11614
11615
11616
11617
11618
11619
11620
11621
11622
11623
11624
11625
11626
11627
11628
11629
11630
11631
11632
11633
11634
11635
11636
11637
11638
11639
11640
11641
11642
11643
11644
11645
11646
11647
11648
11649
11650
11651
11652
11653
11654
11655
11656
11657
11658
11659
11660
11661
11662
11663
11664
11665
11666
11667
11668
11669
11670
11671
11672
11673
11674
11675
11676
11677
11678
11679
11680
11681
11682
11683
11684
11685
11686
11687
11688
11689
11690
def unbind(self, preserve_shared_data: bool = False):
    """
    Unbind this agent from any binding configuration.

    Args:
        preserve_shared_data: Whether to preserve shared data in the agent after unbinding

    Returns:
        dict: Unbinding result with statistics
    """
    if not self.shared.get('is_bound', False):
        return {
            'success': False,
            'message': f"Agent {self.amd.name} is not currently bound to any other agents"
        }

    binding_config = self.shared.get('binding_config')
    if not binding_config:
        return {
            'success': False,
            'message': "No binding configuration found"
        }

    binding_id = binding_config['binding_id']
    bound_agents = binding_config['agents']

    unbind_stats = {
        'binding_id': binding_id,
        'agents_affected': [],
        'shared_data_preserved': preserve_shared_data,
        'private_data_restored': False,
        'unbind_timestamp': datetime.now().isoformat()
    }

    try:
        # Restore original managers for this agent
        if hasattr(self, '_original_managers'):
            original = self._original_managers

            if preserve_shared_data:
                # Merge current shared data with original data
                if isinstance(original['world_model'], dict):
                    original['world_model'].update(self.world_model)
                if isinstance(original['shared'], dict):
                    original['shared'].update({k: v for k, v in self.shared.items()
                                               if k not in ['binding_config', 'is_bound', 'binding_id',
                                                            'bound_agents']})

            # Restore original variable manager
            self.variable_manager = original['variable_manager']
            self.world_model = original['world_model']
            self.shared = original['shared']

            # Update context manager
            if hasattr(self, 'context_manager') and self.context_manager:
                self.context_manager.variable_manager = self.variable_manager

            unbind_stats['private_data_restored'] = True
            del self._original_managers

        # Clean up binding state
        self.shared.pop('binding_config', None)
        self.shared.pop('is_bound', None)
        self.shared.pop('binding_id', None)
        self.shared.pop('bound_agents', None)

        # Update binding configuration to remove this agent
        remaining_agents = [agent for agent in bound_agents if agent != self]
        if remaining_agents:
            # Update binding config for remaining agents
            binding_config['agents'] = remaining_agents
            for agent in remaining_agents:
                if hasattr(agent, 'shared') and agent.shared.get('is_bound'):
                    agent.shared['bound_agents'] = [a.amd.name for a in remaining_agents]

        unbind_stats['agents_affected'] = [agent.amd.name for agent in bound_agents]

        # Clean up sync handler if this was the last agent
        if len(remaining_agents) <= 1:
            sync_handler = binding_config.get('sync_handler')
            if sync_handler and hasattr(sync_handler, 'cleanup'):
                sync_handler.cleanup()

        rprint(f"Agent {self.amd.name} successfully unbound from binding {binding_id}")
        rprint(f"Shared data preserved: {preserve_shared_data}")

        return {
            'success': True,
            'stats': unbind_stats,
            'message': f"Agent {self.amd.name} unbound successfully"
        }

    except Exception as e:
        eprint(f"Error during unbinding: {e}")
        return {
            'success': False,
            'error': str(e),
            'stats': unbind_stats
        }
FormatConfig dataclass

Konfiguration für Response-Format und -Länge

Source code in toolboxv2/mods/isaa/base/Agent/types.py
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
@dataclass
class FormatConfig:
    """Konfiguration für Response-Format und -Länge"""
    response_format: ResponseFormat = ResponseFormat.FREE_TEXT
    text_length: TextLength = TextLength.CHAT_CONVERSATION
    custom_instructions: str = ""
    strict_format_adherence: bool = True
    quality_threshold: float = 0.7

    def get_format_instructions(self) -> str:
        """Generiere Format-spezifische Anweisungen"""
        format_instructions = {
            ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
            ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
            ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
            ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
            ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
            ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
            ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
            ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
            ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
            ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
        }
        return format_instructions.get(self.response_format, "Standard-Formatierung.")

    def get_length_instructions(self) -> str:
        """Generiere Längen-spezifische Anweisungen"""
        length_instructions = {
            TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
            TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
            TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
            TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
            TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
        }
        return length_instructions.get(self.text_length, "Standard-Länge.")

    def get_combined_instructions(self) -> str:
        """Kombiniere Format- und Längen-Anweisungen"""
        instructions = []
        instructions.append("## Format-Anforderungen:")
        instructions.append(self.get_format_instructions())
        instructions.append("\n## Längen-Anforderungen:")
        instructions.append(self.get_length_instructions())

        if self.custom_instructions:
            instructions.append("\n## Zusätzliche Anweisungen:")
            instructions.append(self.custom_instructions)

        if self.strict_format_adherence:
            instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

        return "\n".join(instructions)

    def get_expected_word_range(self) -> tuple[int, int]:
        """Erwartete Wortanzahl für Qualitätsbewertung"""
        ranges = {
            TextLength.MINI_CHAT: (10, 50),
            TextLength.CHAT_CONVERSATION: (50, 150),
            TextLength.TABLE_CONVERSATION: (100, 250),
            TextLength.DETAILED_INDEPTH: (300, 800),
            TextLength.PHD_LEVEL: (800, 2000)
        }
        return ranges.get(self.text_length, (50, 200))
get_combined_instructions()

Kombiniere Format- und Längen-Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
def get_combined_instructions(self) -> str:
    """Kombiniere Format- und Längen-Anweisungen"""
    instructions = []
    instructions.append("## Format-Anforderungen:")
    instructions.append(self.get_format_instructions())
    instructions.append("\n## Längen-Anforderungen:")
    instructions.append(self.get_length_instructions())

    if self.custom_instructions:
        instructions.append("\n## Zusätzliche Anweisungen:")
        instructions.append(self.custom_instructions)

    if self.strict_format_adherence:
        instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

    return "\n".join(instructions)
get_expected_word_range()

Erwartete Wortanzahl für Qualitätsbewertung

Source code in toolboxv2/mods/isaa/base/Agent/types.py
416
417
418
419
420
421
422
423
424
425
def get_expected_word_range(self) -> tuple[int, int]:
    """Erwartete Wortanzahl für Qualitätsbewertung"""
    ranges = {
        TextLength.MINI_CHAT: (10, 50),
        TextLength.CHAT_CONVERSATION: (50, 150),
        TextLength.TABLE_CONVERSATION: (100, 250),
        TextLength.DETAILED_INDEPTH: (300, 800),
        TextLength.PHD_LEVEL: (800, 2000)
    }
    return ranges.get(self.text_length, (50, 200))
get_format_instructions()

Generiere Format-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
def get_format_instructions(self) -> str:
    """Generiere Format-spezifische Anweisungen"""
    format_instructions = {
        ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
        ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
        ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
        ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
        ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
        ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
        ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
        ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
        ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
        ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
    }
    return format_instructions.get(self.response_format, "Standard-Formatierung.")
get_length_instructions()

Generiere Längen-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py
388
389
390
391
392
393
394
395
396
397
def get_length_instructions(self) -> str:
    """Generiere Längen-spezifische Anweisungen"""
    length_instructions = {
        TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
        TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
        TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
        TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
        TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
    }
    return length_instructions.get(self.text_length, "Standard-Länge.")
LLMReasonerNode

Bases: AsyncNode

Enhanced strategic reasoning core with outline-driven execution, context management, auto-recovery, and intensive variable system integration.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
6589
6590
6591
6592
6593
6594
6595
6596
6597
6598
6599
6600
6601
6602
6603
6604
6605
6606
6607
6608
6609
6610
6611
6612
6613
6614
6615
6616
6617
6618
6619
6620
6621
6622
6623
6624
6625
6626
6627
6628
6629
6630
6631
6632
6633
6634
6635
6636
6637
6638
6639
6640
6641
6642
6643
6644
6645
6646
6647
6648
6649
6650
6651
6652
6653
6654
6655
6656
6657
6658
6659
6660
6661
6662
6663
6664
6665
6666
6667
6668
6669
6670
6671
6672
6673
6674
6675
6676
6677
6678
6679
6680
6681
6682
6683
6684
6685
6686
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
6735
6736
6737
6738
6739
6740
6741
6742
6743
6744
6745
6746
6747
6748
6749
6750
6751
6752
6753
6754
6755
6756
6757
6758
6759
6760
6761
6762
6763
6764
6765
6766
6767
6768
6769
6770
6771
6772
6773
6774
6775
6776
6777
6778
6779
6780
6781
6782
6783
6784
6785
6786
6787
6788
6789
6790
6791
6792
6793
6794
6795
6796
6797
6798
6799
6800
6801
6802
6803
6804
6805
6806
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818
6819
6820
6821
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836
6837
6838
6839
6840
6841
6842
6843
6844
6845
6846
6847
6848
6849
6850
6851
6852
6853
6854
6855
6856
6857
6858
6859
6860
6861
6862
6863
6864
6865
6866
6867
6868
6869
6870
6871
6872
6873
6874
6875
6876
6877
6878
6879
6880
6881
6882
6883
6884
6885
6886
6887
6888
6889
6890
6891
6892
6893
6894
6895
6896
6897
6898
6899
6900
6901
6902
6903
6904
6905
6906
6907
6908
6909
6910
6911
6912
6913
6914
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926
6927
6928
6929
6930
6931
6932
6933
6934
6935
6936
6937
6938
6939
6940
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952
6953
6954
6955
6956
6957
6958
6959
6960
6961
6962
6963
6964
6965
6966
6967
6968
6969
6970
6971
6972
6973
6974
6975
6976
6977
6978
6979
6980
6981
6982
6983
6984
6985
6986
6987
6988
6989
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999
7000
7001
7002
7003
7004
7005
7006
7007
7008
7009
7010
7011
7012
7013
7014
7015
7016
7017
7018
7019
7020
7021
7022
7023
7024
7025
7026
7027
7028
7029
7030
7031
7032
7033
7034
7035
7036
7037
7038
7039
7040
7041
7042
7043
7044
7045
7046
7047
7048
7049
7050
7051
7052
7053
7054
7055
7056
7057
7058
7059
7060
7061
7062
7063
7064
7065
7066
7067
7068
7069
7070
7071
7072
7073
7074
7075
7076
7077
7078
7079
7080
7081
7082
7083
7084
7085
7086
7087
7088
7089
7090
7091
7092
7093
7094
7095
7096
7097
7098
7099
7100
7101
7102
7103
7104
7105
7106
7107
7108
7109
7110
7111
7112
7113
7114
7115
7116
7117
7118
7119
7120
7121
7122
7123
7124
7125
7126
7127
7128
7129
7130
7131
7132
7133
7134
7135
7136
7137
7138
7139
7140
7141
7142
7143
7144
7145
7146
7147
7148
7149
7150
7151
7152
7153
7154
7155
7156
7157
7158
7159
7160
7161
7162
7163
7164
7165
7166
7167
7168
7169
7170
7171
7172
7173
7174
7175
7176
7177
7178
7179
7180
7181
7182
7183
7184
7185
7186
7187
7188
7189
7190
7191
@with_progress_tracking
class LLMReasonerNode(AsyncNode):
    """
    Enhanced strategic reasoning core with outline-driven execution,
    context management, auto-recovery, and intensive variable system integration.
    """

    def __init__(self, max_reasoning_loops: int = 24, **kwargs):
        super().__init__(**kwargs)
        self.max_reasoning_loops = max_reasoning_loops
        self.reasoning_context = []
        self.internal_task_stack = []
        self.meta_tools_registry = {}
        self.current_loop_count = 0
        self.current_reasoning_count = 0
        self.agent_instance: FlowAgent = None

        # Enhanced tracking systems
        self.outline = None
        self.current_outline_step = 0
        self.step_completion_tracking = {}
        self.loop_detection_memory = []
        self.context_summary_threshold = 15
        self.max_context_size = 30
        self.performance_metrics = {
                "loop_times": [],
                "progress_loops": 0,
                "total_loops": 0
            }
        self.auto_recovery_attempts = 0
        self.max_auto_recovery = 8
        self.variable_manager = None

        # Anti-loop mechanisms
        self.last_action_signatures = []
        self.step_enforcement_active = True
        self.mandatory_progress_check = True

    async def prep_async(self, shared):
        """Enhanced initialization with variable system integration"""
        # Reset for new execution
        self.reasoning_context = []
        self.internal_task_stack = []
        self.current_loop_count = 0
        self.current_reasoning_count = 0
        self.outline = None
        self.current_outline_step = 0
        self.step_completion_tracking = {}
        self.loop_detection_memory = []
        self.performance_metrics = {
            "loop_times": [],
            "progress_loops": 0,
            "total_loops": 0
        }
        self.auto_recovery_attempts = 0
        self.last_action_signatures = []

        self.agent_instance = shared.get("agent_instance")

        # Enhanced variable manager integration
        self.variable_manager = shared.get("variable_manager", self.agent_instance.variable_manager)
        context_manager = shared.get("context_manager")

        if self.variable_manager:
            # Store reasoning session context
            session_context = {
                "session_id": shared.get("session_id", "default"),
                "start_time": datetime.now().isoformat(),
                "query": shared.get("current_query", ""),
                "reasoning_mode": "outline_driven"
            }
            self.variable_manager.set("reasoning.current_session", session_context)
            # Load previous successful patterns from variables
            self._load_historical_patterns()

        #Build comprehensive system context via UnifiedContextManager
        system_context = await self._build_enhanced_system_context_unified(shared, context_manager)

        return {
            "original_query": shared.get("current_query", ""),
            "session_id": shared.get("session_id", "default"),
            "agent_instance": shared.get("agent_instance"),
            "variable_manager": self.variable_manager,
            "context_manager": context_manager,  #Context Manager Reference
            "system_context": system_context,
            "available_tools": shared.get("available_tools", []),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "progress_tracker": shared.get("progress_tracker"),
            "formatted_context": shared.get("formatted_context", {}),
            "historical_context": await self._get_historical_context_unified(context_manager, shared.get("session_id")),
            "capabilities_summary": shared.get("capabilities_summary", ""),
            # Sub-system references
            "llm_tool_node": shared.get("llm_tool_node_instance"),
            "task_planner": shared.get("task_planner_instance"),
            "task_executor": shared.get("task_executor_instance"),
        }

    async def exec_async(self, prep_res):
        """Enhanced main reasoning loop with outline-driven execution"""
        if not LITELLM_AVAILABLE:
            return await self._fallback_direct_response(prep_res)

        original_query = prep_res["original_query"]
        agent_instance = prep_res["agent_instance"]
        progress_tracker = prep_res.get("progress_tracker")

        # Initialize enhanced reasoning context
        await self._initialize_reasoning_session(prep_res, original_query)

        # STEP 1: MANDATORY OUTLINE CREATION
        if not self.outline:
            with Spinner("Creating initial outline..."):
                outline_result = await self._create_initial_outline(prep_res)
            if self.outline and len(self.outline.get("steps", [])) == 1:
                # fast llm respose on the input metoning tis is a direct respose and evalute if the input dosent need an outline
                print("Fast direct response triggered")
                response = await self.agent_instance.a_run_llm_completion(
                    model=prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku"),
                    messages=[{"role": "user", "content": prep_res["original_query"]}],
                    temperature=0.3,
                    max_tokens=2048,
                    node_name="LLMReasonerNode",
                    task_id="fast_direct_response"
                )
                return {
                        "final_result": response,
                        "reasoning_loops": self.current_loop_count,
                        "reasoning_context": self.reasoning_context.copy(),
                        "internal_task_stack": self.internal_task_stack.copy(),
                        "outline": self.outline,
                        "outline_completion": self.current_outline_step,
                        "performance_metrics": self.performance_metrics,
                        "auto_recovery_attempts": self.auto_recovery_attempts
                    }
            elif not outline_result:
                return await self._fallback_direct_response(prep_res)

        final_result = None
        consecutive_no_progress = 0
        max_no_progress = 3

        # Enhanced main reasoning loop with strict progress tracking
        while self.current_reasoning_count < self.max_reasoning_loops:
            self.current_loop_count += 1
            loop_start_time = time.time()

            # Check for infinite loops
            if self._detect_infinite_loop():
                await self._trigger_auto_recovery(prep_res)
                if self.auto_recovery_attempts >= self.max_auto_recovery:
                    break

            # Auto-context management
            await self._manage_context_size()

            # Progress tracking
            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="reasoning_loop",
                    timestamp=time.time(),
                    node_name="LLMReasonerNode",
                    status=NodeStatus.RUNNING,
                    metadata={
                        "loop_number": self.current_loop_count,
                        "outline_step": self.current_outline_step,
                        "outline_total": len(self.outline.get("steps", [])) if self.outline else 0,
                        "context_size": len(self.reasoning_context),
                        "task_stack_size": len(self.internal_task_stack),
                        "auto_recovery_attempts": self.auto_recovery_attempts,
                        "performance_metrics": self.performance_metrics
                    }
                ))

            try:
                # Build enhanced reasoning prompt with outline context
                reasoning_prompt = await self._build_outline_driven_prompt(prep_res)

                # Force progress check if needed
                if self.mandatory_progress_check and consecutive_no_progress >= 2:
                    reasoning_prompt += "\n\n**MANDATORY**: You must either complete current outline step or move to next step. No more analysis without action!"

                # LLM reasoning call
                model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")

                llm_response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=[{"role": "user", "content": reasoning_prompt}],
                    temperature=0.2,  # Lower temperature for more focused execution
                    # max_tokens=3072,
                    node_name="LLMReasonerNode",
                    stop="<immediate_context>",
                    task_id=f"reasoning_loop_{self.current_loop_count}_step_{self.current_outline_step}"
                )

                # Add LLM response to context
                self.reasoning_context.append({
                    "type": "reasoning",
                    "content": llm_response,
                    "loop": self.current_loop_count,
                    "outline_step": self.current_outline_step,
                    "timestamp": datetime.now().isoformat()
                })

                # Parse and execute meta-tool calls with enhanced tracking
                progress_made = await self._parse_and_execute_meta_tools(llm_response, prep_res)

                action_taken = progress_made.get("action_taken", False)
                actual_progress = progress_made.get("progress_made", False)

                # Update performance with correct progress indication
                self._update_performance_metrics(loop_start_time, actual_progress)

                if not action_taken:
                    self.current_reasoning_count += 1
                    if self.current_outline_step > len(self.outline.get("steps", [])):
                        progress_made["final_result"] = llm_response
                        rprint("Final result reached forced by outline step count")
                    if self.current_outline_step < len(self.outline.get("steps", [])) and self.outline.get("steps", [])[self.current_outline_step].get("is_final", False):
                        progress_made["final_result"] = llm_response
                        rprint("Final result reached forced by outline step count final step")
                else:
                    self.current_reasoning_count -= 1

                # Check for final result
                if progress_made.get("final_result"):
                    final_result = progress_made["final_result"]
                    await self._finalize_reasoning_session(prep_res, final_result)
                    break

                # Progress monitoring
                if progress_made.get("action_taken"):
                    consecutive_no_progress = 0
                    self._update_performance_metrics(loop_start_time, True)
                else:
                    consecutive_no_progress += 1
                    self._update_performance_metrics(loop_start_time, False)

                # Check outline completion
                if self.outline and self.current_outline_step >= len(self.outline.get("steps", []))+self.max_reasoning_loops:
                    # All outline steps completed, force final response
                    final_result = await self._create_outline_completion_response(prep_res)
                    break

                # Emergency break for excessive no-progress
                if consecutive_no_progress >= max_no_progress:
                    await self._trigger_auto_recovery(prep_res)

            except Exception as e:
                await self._handle_reasoning_error(e, prep_res, progress_tracker)
                import traceback
                print(traceback.format_exc())
                if self.auto_recovery_attempts >= self.max_auto_recovery:
                    final_result = await self._create_error_response(original_query, str(e))
                    break


        # If no final result after max loops, create a comprehensive summary
        if not final_result:
            final_result = await self._create_enhanced_timeout_response(original_query, prep_res)

        return {
            "final_result": final_result,
            "reasoning_loops": self.current_loop_count,
            "reasoning_context": self.reasoning_context.copy(),
            "internal_task_stack": self.internal_task_stack.copy(),
            "outline": self.outline,
            "outline_completion": self.current_outline_step,
            "performance_metrics": self.performance_metrics,
            "auto_recovery_attempts": self.auto_recovery_attempts
        }

    async def _build_enhanced_system_context_unified(self, shared, context_manager) -> str:
        """Build comprehensive system context mit UnifiedContextManager"""
        context_parts = []

        # Enhanced agent capabilities
        available_tools = shared.get("available_tools", [])
        if available_tools:
            context_parts.append(f"Available external tools: {', '.join(available_tools)}")

        #Context Manager Status
        if context_manager:
            session_stats = context_manager.get_session_statistics()
            context_parts.append(f"Context System: Advanced with {session_stats['total_sessions']} active sessions")
            context_parts.append(f"Cache Status: {session_stats['cache_entries']} cached contexts")

        # Variable system context
        if self.variable_manager:
            var_info = self.variable_manager.get_scope_info()
            context_parts.append(f"Variable System: {len(var_info)} scopes available")

            # Recent results availability
            results_count = len(self.variable_manager.get("results", {}))
            if results_count:
                context_parts.append(f"Previous results: {results_count} task results available")

        #Enhanced system state mit Context-Awareness
        session_id = shared.get("session_id", "default")
        if context_manager and session_id in context_manager.session_managers:
            session = context_manager.session_managers[session_id]
            if hasattr(session, 'history'):
                context_parts.append(f"Session History: {len(session.history)} conversation entries available")
            elif isinstance(session, dict) and 'history' in session:
                context_parts.append(f"Session History: {len(session['history'])} conversation entries (fallback mode)")

        # System state with enhanced details
        tasks = shared.get("tasks", {})
        if tasks:
            active_tasks = len([t for t in tasks.values() if t.status == "running"])
            completed_tasks = len([t for t in tasks.values() if t.status == "completed"])
            context_parts.append(f"Execution state: {active_tasks} active, {completed_tasks} completed tasks")

        # Performance history
        if hasattr(self, 'historical_successful_patterns'):
            context_parts.append(
                f"Historical patterns: {len(self.historical_successful_patterns)} successful patterns loaded")

        return "\n".join(context_parts) if context_parts else "Basic system context available"

    async def _get_historical_context_unified(self, context_manager, session_id: str) -> str:
        """Get historical context from UnifiedContextManager"""
        if not context_manager:
            return ""

        try:
            #Get unified context for historical analysis
            unified_context = await context_manager.build_unified_context(session_id, None, "historical")

            context_parts = []

            # Chat history insights
            chat_history = unified_context.get("chat_history", [])
            if chat_history:
                context_parts.append(f"Conversation History: {len(chat_history)} messages available")

                # Analyze conversation patterns
                user_queries = [msg['content'] for msg in chat_history if msg.get('role') == 'user']
                if user_queries:
                    avg_query_length = sum(len(q) for q in user_queries) / len(user_queries)
                    context_parts.append(f"Query patterns: Avg length {avg_query_length:.0f} chars")

            # Execution history from variables
            if self.variable_manager:
                # Recent successful queries
                recent_successes = self.variable_manager.get("reasoning.recent_successes", [])
                if recent_successes:
                    context_parts.append(f"Recent successful queries: {len(recent_successes)}")

                # Performance history
                avg_loops = self.variable_manager.get("reasoning.performance.avg_loops", 0)
                if avg_loops:
                    context_parts.append(f"Average reasoning loops: {avg_loops}")

            # System insights from unified context
            execution_state = unified_context.get("execution_state", {})
            if execution_state.get("recent_completions"):
                completions = execution_state["recent_completions"]
                context_parts.append(f"Recent completions: {len(completions)} tasks finished")

            return "\n".join(context_parts)

        except Exception as e:
            eprint(f"Failed to get historical context: {e}")
            return "Historical context unavailable"

    def _load_historical_patterns(self):
        """Load successful patterns from previous reasoning sessions"""
        if not self.variable_manager:
            return

        # Load successful outline patterns
        successful_outlines = self.variable_manager.get("reasoning.successful_patterns.outlines", [])
        failed_patterns = self.variable_manager.get("reasoning.failed_patterns", [])

        self.historical_successful_patterns = successful_outlines[-5:]  # Last 5 successful
        self.historical_failed_patterns = failed_patterns[-10:]  # Last 10 failed

    def _get_historical_context(self) -> str:
        """Get historical context from variable system"""
        if not self.variable_manager:
            return ""

        context_parts = []

        # Recent successful queries
        recent_successes = self.variable_manager.get("reasoning.recent_successes", [])
        if recent_successes:
            context_parts.append(f"Recent successful queries: {len(recent_successes)}")

        # Performance history
        avg_loops = self.variable_manager.get("reasoning.performance.avg_loops", 0)
        if avg_loops:
            context_parts.append(f"Average reasoning loops: {avg_loops}")

        # Common failure patterns to avoid
        failure_patterns = self.variable_manager.get("reasoning.failure_patterns", [])
        if failure_patterns:
            context_parts.append(f"Known failure patterns: {len(failure_patterns)}")

        return "\n".join(context_parts)

    async def _initialize_reasoning_session(self, prep_res, original_query):
        """Initialize enhanced reasoning session with variable tracking"""
        # Initialize reasoning context
        self.reasoning_context.append({
            "type": "session_start",
            "content": f"Enhanced reasoning session started for: {original_query}",
            "timestamp": datetime.now().isoformat(),
            "session_id": prep_res.get("session_id")
        })

        # Store session in variables
        if self.variable_manager:
            session_data = {
                "query": original_query,
                "start_time": datetime.now().isoformat(),
                "max_loops": self.max_reasoning_loops,
                "context_management": "auto_summary",
                "outline_driven": True
            }
            self.variable_manager.set("reasoning.current_session.data", session_data)

        # Add enhanced system context
        self.reasoning_context.append({
            "type": "system_context",
            "content": prep_res["system_context"],
            "timestamp": datetime.now().isoformat()
        })

        # Add historical context if available
        historical = prep_res.get("historical_context")
        if historical:
            self.reasoning_context.append({
                "type": "historical_context",
                "content": historical,
                "timestamp": datetime.now().isoformat()
            })

    async def _create_initial_outline(self, prep_res) -> bool:
        """Create mandatory initial outline, with a fast path for simple queries."""
        original_query = prep_res["original_query"]
        agent_instance = prep_res["agent_instance"]

        outline_prompt = f"""You MUST create an initial execution outline for this query. This is mandatory.

**Query:** {original_query}

**Available Resources:**
- Tools: {', '.join(prep_res.get('available_tools', []))}
- Sub-systems: LLM Tool Node, Task Planner, Task Executor

LLM Tool Node is for all tool calls!
LLM Tool Node is best for simple multi-step tasks like fetching data from a tool and summarizing it.
Task Planner is best for complex tasks with multiple dependencies and complex task flows.

**Historical Context:** {prep_res.get('historical_context', 'None')}

**Fast Path for Simple Queries:**
If the query is simple and can be answered directly without needing tools or complex reasoning, you MUST create a single-step outline using the `direct_response` method.

Create a structured outline using this EXACT format:

```outline
OUTLINE_START
Step 1: [Brief description of first step]
- Method: [internal_reasoning | delegate_to_llm_tool_node | direct_response]
- Expected outcome: [What this step should achieve]
- Success criteria: [How to know this step is complete]

[For complex queries, continue with more steps as needed.]

Final Step: Synthesize results and provide comprehensive response
- Method: direct_response
- Expected outcome: Complete answer to user query
- Success criteria: User query fully addressed
OUTLINE_END
```

**Requirements:**
1. Outline must have between 1 and 7 steps.
2. For simple queries, a single "Final Step" using the 'direct_response' method is the correct approach.
3. Each step must have clear success criteria and build logically toward the answer.
4. Be specific about which meta-tools to use for each step. meta-tools ar not Tools ! avalabel meta-tools *Method* (internal_reasoning, delegate_to_llm_tool_node, direct_response) no exceptions

Create the outline now:"""

        try:
            llm_response = await agent_instance.a_run_llm_completion(
                model=prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o"),
                messages=[{"role": "system", "content": outline_prompt}, {"role": "user", "content": original_query}],
                temperature=0.2,  # Lower temperature for more deterministic outlining
                node_name="LLMReasonerNode",
                task_id="create_initial_outline",
                stream=False,
                auto_fallbacks=False,
                stop=["OUTLINE_END"]
            )
            llm_response += "OUTLINE_END"

            # Parse outline from response
            # print(llm_response)
            outline = self._parse_outline_from_response(llm_response)

            if self.agent_instance and self.agent_instance.progress_tracker:
                await self.agent_instance.progress_tracker.emit_event(ProgressEvent(
                    event_type="outline_created",
                    timestamp=time.time(),
                    node_name="LLMReasonerNode",
                    status=NodeStatus.COMPLETED,
                    task_id="create_initial_outline",
                    metadata={"outline": outline}
                ))

            if outline:
                self.outline = outline
                self.current_outline_step = 0

                # Store outline in variables
                if self.variable_manager:
                    self.variable_manager.set("reasoning.current_session.outline", outline)

                # Add to reasoning context
                self.reasoning_context.append({
                    "type": "outline_created",
                    "content": f"Created outline with {len(outline.get('steps', []))} steps",
                    "outline": outline,
                    "timestamp": datetime.now().isoformat()
                })

                return True
            else:
                return False

        except Exception as e:
            eprint(f"Failed to create initial outline: {e}")
            import traceback
            print(traceback.format_exc())
            return False

    def _parse_outline_from_response(self, response: str) -> dict[str, Any]:
        """Parse structured outline from LLM response"""
        import re

        # Find outline section
        outline_match = re.search(r'OUTLINE_START(.*?)OUTLINE_END', response, re.DOTALL)
        if not outline_match:
            return None

        outline_text = outline_match.group(1).strip()

        # Parse steps
        steps = []
        current_step = None

        for line in outline_text.split('\n'):
            line = line.strip()
            if not line:
                continue

            # New step
            if re.match(r'^Step \d+:', line):
                if current_step:
                    steps.append(current_step)

                current_step = {
                    "description": re.sub(r'^Step \d+:\s*', '', line),
                    "method": "",
                    "expected_outcome": "",
                    "success_criteria": "",
                    "status": "pending"
                }
            elif re.match(r'^Final Step:', line):
                if current_step:
                    steps.append(current_step)

                current_step = {
                    "description": re.sub(r'^Final Step:\s*', '', line),
                    "method": "direct_response",
                    "expected_outcome": "",
                    "success_criteria": "",
                    "status": "pending",
                    "is_final": True
                }
            elif current_step and line.startswith('- Method:'):
                current_step["method"] = line.replace('- Method:', '').strip()
            elif current_step and line.startswith('- Expected outcome:'):
                current_step["expected_outcome"] = line.replace('- Expected outcome:', '').strip()
            elif current_step and line.startswith('- Success criteria:'):
                current_step["success_criteria"] = line.replace('- Success criteria:', '').strip()

        # Add final step if exists
        if current_step:
            steps.append(current_step)

        if not steps:
            return None

        return {
            "steps": steps,
            "created_at": datetime.now().isoformat(),
            "total_steps": len(steps)
        }

    def _build_enhanced_system_context(self, shared) -> str:
        """Build comprehensive system context with variable system info"""
        context_parts = []

        # Enhanced agent capabilities
        available_tools = shared.get("available_tools", [])
        if available_tools:
            context_parts.append(f"Available external tools: {', '.join(available_tools)}")

        # Variable system context
        if self.variable_manager:
            var_info = self.variable_manager.get_scope_info()
            context_parts.append(f"Variable System: {len(var_info)} scopes available")

            # Recent results availability
            results_count = len(self.variable_manager.get("results", {}))
            if results_count:
                context_parts.append(f"Previous results: {results_count} task results available")

        # System state with enhanced details
        tasks = shared.get("tasks", {})
        if tasks:
            active_tasks = len([t for t in tasks.values() if t.status == "running"])
            completed_tasks = len([t for t in tasks.values() if t.status == "completed"])
            context_parts.append(f"Execution state: {active_tasks} active, {completed_tasks} completed tasks")

        # Session context with history
        formatted_context = shared.get("formatted_context", {})
        if formatted_context:
            recent_interaction = formatted_context.get("recent_interaction", "")
            if recent_interaction:
                context_parts.append(f"Recent interaction: {recent_interaction[:100000]}...")

        # Performance history
        if hasattr(self, 'historical_successful_patterns'):
            context_parts.append(
                f"Historical patterns: {len(self.historical_successful_patterns)} successful patterns loaded")

        return "\n".join(context_parts) if context_parts else "Basic system context available"

    async def _manage_context_size(self):
        """Auto-manage context size with intelligent summarization"""
        if len(self.reasoning_context) <= self.context_summary_threshold:
            return

        # Trigger summarization
        if len(self.reasoning_context) >= self.max_context_size:
            # Emergency summarization
            await self._emergency_context_summary()
        elif len(self.reasoning_context) >= self.context_summary_threshold:
            # Regular summarization
            await self._regular_context_summary()

    async def _regular_context_summary(self):
        """Regular context summarization when threshold is reached"""
        # Keep last 10 entries, summarize the rest
        keep_recent = self.reasoning_context[-10:]
        to_summarize = self.reasoning_context[:-10]

        summary = self._create_context_summary(to_summarize, "regular")

        # Replace old context with summary + recent
        self.reasoning_context = [
                                     {
                                         "type": "context_summary",
                                         "content": summary,
                                         "summarized_entries": len(to_summarize),
                                         "summary_type": "regular",
                                         "timestamp": datetime.now().isoformat()
                                     }
                                 ] + keep_recent

    async def _emergency_context_summary(self):
        """Emergency context summarization when max size is reached"""
        # Keep last 5 entries, summarize everything else
        keep_recent = self.reasoning_context[-5:]
        to_summarize = self.reasoning_context[:-5]

        summary = self._create_context_summary(to_summarize, "emergency")

        # Replace with emergency summary
        self.reasoning_context = [
                                     {
                                         "type": "context_summary",
                                         "content": summary,
                                         "summarized_entries": len(to_summarize),
                                         "summary_type": "emergency",
                                         "timestamp": datetime.now().isoformat()
                                     }
                                 ] + keep_recent

    def _create_context_summary(self, entries: list[dict], summary_type: str) -> str:
        """Create intelligent context summary"""
        if not entries:
            return "No context to summarize"

        summary_parts = []

        # Group by type
        by_type = {}
        for entry in entries:
            entry_type = entry.get("type", "unknown")
            if entry_type not in by_type:
                by_type[entry_type] = []
            by_type[entry_type].append(entry)

        # Summarize each type
        for entry_type, type_entries in by_type.items():
            if entry_type == "reasoning":
                reasoning_summary = f"Completed {len(type_entries)} reasoning cycles"
                # Extract key insights
                insights = []
                for entry in type_entries[-3:]:  # Last 3 reasoning entries
                    content = entry.get("content", "")[:1000] + "..."
                    insights.append(content)
                if insights:
                    reasoning_summary += f"\nKey recent reasoning: {'; '.join(insights)}"
                summary_parts.append(reasoning_summary)

            elif entry_type == "meta_tool_result":
                results_summary = f"Executed {len(type_entries)} meta-tool operations"
                # Extract significant results
                significant_results = [
                    entry.get("content", "")[:800]
                    for entry in type_entries
                    if len(entry.get("content", "")) > 50
                ]
                if significant_results:
                    results_summary += f"\nSignificant results: {'; '.join(significant_results[-3:])}"
                summary_parts.append(results_summary)

            else:
                summary_parts.append(f"{entry_type}: {len(type_entries)} entries")

        summary = f"[{summary_type.upper()} SUMMARY] " + "; ".join(summary_parts)

        # Store summary in variables for future reference
        if self.variable_manager:
            summary_data = {
                "type": summary_type,
                "entries_summarized": len(entries),
                "summary": summary,
                "timestamp": datetime.now().isoformat()
            }
            summaries = self.variable_manager.get("reasoning.context_summaries", [])
            summaries.append(summary_data)
            self.variable_manager.set("reasoning.context_summaries", summaries[-10:])  # Keep last 10

        return summary

    def _get_pending_tasks_summary(self) -> str:
        """Get summary of pending tasks requiring attention"""
        if not self.internal_task_stack:
            return "⚠️ NO TASKS IN STACK - You must create tasks from your outline immediately!"

        pending_tasks = [task for task in self.internal_task_stack if task.get("status", "pending") == "pending"]

        if not pending_tasks:
            return "✅ No pending tasks - ready for next outline step or completion"

        task_summaries = []
        for i, task in enumerate(pending_tasks[:3], 1):
            desc = task.get("description", "No description")[:150] + "..." if len(
                task.get("description", "")) > 50 else task.get("description", "")
            step_ref = task.get("outline_step_ref", "")
            step_info = f" ({step_ref})" if step_ref else ""
            task_summaries.append(f"{i}. {desc}{step_info}")

        if len(pending_tasks) > 3:
            task_summaries.append(f"... +{len(pending_tasks) - 3} more pending tasks")

        return f"📋 {len(pending_tasks)} pending tasks:\n" + "\n".join(task_summaries)

    async def _build_outline_driven_prompt(self, prep_res) -> str:
        """Build outline-driven reasoning prompt mit UnifiedContextManager Integration"""

        # Get current task with enhanced visibility
        current_stack_task = self._get_current_stack_task()

        #Enhanced context aus UnifiedContextManager
        context_manager = prep_res.get("context_manager")
        session_id = prep_res.get("session_id", "default")

        # Build unified context sections
        unified_context_summary = ""
        recent_results_context = ""

        if context_manager:
            try:
                # Get full unified context
                unified_context = await  context_manager.build_unified_context(session_id, prep_res.get('original_query'))

                unified_context_summary = self._format_unified_context_for_reasoning(unified_context)
                recent_results_context = self._build_recent_results_from_unified_context(unified_context)
            except Exception as e:
                eprint(f"Failed to get unified context in reasoning prompt: {e}")
                unified_context_summary = "Unified context unavailable"
                recent_results_context = "**No recent results available**"

        # Enhanced context summaries (keeping existing functionality)
        context_summary = self._summarize_reasoning_context()
        task_stack_summary = self._summarize_task_stack()
        outline_status = self._get_current_step_requirements()
        performance_context = self._get_performance_context()

        # Enhanced variable system integration with better suggestions
        variable_context = ""
        variable_suggestions = []
        if self.variable_manager:
            variable_context = self.variable_manager.get_llm_variable_context()
            query_text = prep_res.get('original_query', '')
            if current_stack_task:
                query_text += " " + current_stack_task.get('description', '')
            variable_suggestions = self.variable_manager.get_variable_suggestions(query_text)

        immediate_context = self._get_immediate_context_for_prompt()
        # Detect if we're in a potential loop situation
        loop_warning = self._generate_loop_warning()

        prompt = f"""You are the enhanced strategic reasoning core operating in OUTLINE-DRIVEN MODE with MANDATORY TASK STACK enforcement.
## ABSOLUTE REQUIREMENTS - VIOLATION = IMMEDIATE STOP:
1. **WORK ONLY THROUGH TASK STACK** - No work outside the stack permitted
2. **SEE CURRENT TASK DIRECTLY** - Your current task is shown below
3. **USE VARIABLE SYSTEM** - All results are automatically stored and accessible
4. **USE UNIFIED CONTEXT** - Rich conversation and execution history is available
5. **MARK TASKS COMPLETE** - Every finished task must be marked complete
6. **NO REPEATED ACTIONS** - Check variables first before re-doing work

{loop_warning}

## <CURRENT SITUATION>:
**Original Query:** {prep_res['original_query']}

**Unified Context Summary:**
{unified_context_summary}

**Current Context Summary:**
{context_summary}

**Current Outline Status:**
{outline_status}

** CURRENT TASK FROM STACK:**
{current_stack_task}

**Internal Task Stack:**
{task_stack_summary}

**Performance Metrics:**
{performance_context}

## ENHANCED CONTEXT INTEGRATION:
{variable_context}

** SUGGESTED VARIABLES for current task:**
{', '.join(variable_suggestions[:10]) if variable_suggestions else 'tool_capabilities, query, model_complex, available_tools, timestamp, use_fast_response, tool_registry, name, current_query, current_session'}

** UNIFIED CONTEXT RESULTS ACCESS:**
{recent_results_context}

</CURRENT SITUATION>

## MANDATORY TASK STACK ENFORCEMENT:
**CRITICAL RULE**: You MUST work exclusively through your internal task stack.

**TASK STACK WORKFLOW (MANDATORY):**
1. **CHECK CURRENT TASK**: Your current task is: {current_stack_task.get('description', 'NO CURRENT TASK - ADD TASKS FROM OUTLINE!') if current_stack_task else 'NO CURRENT TASK - VIOLATION!'}

2. **WORK ONLY ON STACK TASKS**: You can ONLY work on tasks that exist in your internal task stack
   - The task you're working on MUST be in the stack with status "pending"
   - Before any action: Verify the task exists in your stack

3. **MANDATORY TASK COMPLETION**: After completing any work, you MUST mark the task as complete
   - Use: META_TOOL_CALL: manage_internal_task_stack(action="complete", task_description="[exact task description]", outline_step_ref="step_X")

4. **CHECK UNIFIED CONTEXT FIRST**: Before any major action, focus your attention to the variable system to see if results already exist
   - Avalabel results are automatically stored in the variable system
   - The unified context above shows available conversation history and execution state

**CURRENT TASK ANALYSIS:**
{self._analyze_current_task(current_stack_task) if current_stack_task else "❌ NO CURRENT TASK - You must add tasks from your outline!"}

## AVAILABLE META-TOOLS:
You have access to these meta-tools to control sub-systems. Use the EXACT syntax shown:
{self.meta_tools_registry if self.meta_tools_registry else ''}

**META_TOOL_CALL: internal_reasoning(thought: str, thought_number: int, total_thoughts: int, next_thought_needed: bool, current_focus: str, key_insights: list[str], potential_issues: list[str], confidence_level: float)**
- Purpose: Structure your thinking process explicitly
- Use for: Any complex analysis, planning, or problem decomposition
- Example: META_TOOL_CALL: internal_reasoning(thought="I need to break this down into steps", thought_number=1, total_thoughts=3, next_thought_needed=true, current_focus="problem analysis", key_insights=["Query requires multiple data sources"], potential_issues=["Data might not be available"], confidence_level=0.8)

**META_TOOL_CALL: manage_internal_task_stack(action: str, task_description: str)**
- Purpose: Manage your high-level to-do list
- Actions: "add", "remove", "complete", "get_current"
- Example: META_TOOL_CALL: manage_internal_task_stack(action="add", task_description="Research competitor analysis data")

**META_TOOL_CALL: delegate_to_llm_tool_node(task_description: str, tools_list: list[str])**
- Purpose: Delegate specific, self-contained tasks requiring external tools
- Use for: Web searches, file operations, API calls, single-, two-, or three-step tool usage
- Example: META_TOOL_CALL: delegate_to_llm_tool_node(task_description="Search for latest news about AI developments", tools_list=["search_web"])
- Rule: always validate delegate_to_llm_tool_node result. will be available in <immediate_context> after execution!

**META_TOOL_CALL: read_from_variables(scope: str, key: str, purpose: str)**
- Unified context data is available in various scopes
- Example: META_TOOL_CALL: read_from_variables(scope="user", key="name", purpose="Gather user information for later reference")

**META_TOOL_CALL: write_to_variables(scope: str, key: str, value: any, description: str)**
- Store important findings immediately
- Example: META_TOOL_CALL: write_to_variables(scope="user", key="name", value="User-Name", description="The users name for later reference")

**META_TOOL_CALL: advance_outline_step(step_completed: bool, completion_evidence: str, next_step_focus: str)**
- Mark outline steps complete when all related tasks done

**META_TOOL_CALL: direct_response(final_answer: str, outline_completion: bool, steps_completed: list[str])**
- ONLY when ALL outline steps complete or no META_TOOL_CALL needed
- final_answer must contain the full final answer for the user with all necessary context and informations ( format in persona style )
- Purpose: End reasoning and provide final answer to user
- Use when: Query is complete or can be answered directly
- Example: META_TOOL_CALL: direct_response(final_answer="Based on my analysis, here are the key findings...")

note: in this interaction only META_TOOL_CALL ar avalabel. for other tools use META_TOOL_CALL: delegate_to_llm_tool_node with the appropriate tool names!

## REASONING STRATEGY:
1. **Start with internal_reasoning** to understand the query and plan approach
2. **Use manage_internal_task_stack** to track high-level steps
3. **Choose the right delegation strategy:**
   - Simple queries → direct_response
   - Up to 3 tool tasks with llm action → delegate_to_llm_tool_node
   - Complex projects → create_and_execute_plan
4. **Monitor progress** and adapt your approach
5. **End with direct_response** when complete

## EXAMPLES OF GOOD REASONING PATTERNS:

**Simple Query Pattern:**
META_TOOL_CALL: internal_reasoning(thought="This is a straightforward question I can answer directly", thought_number=1, total_thoughts=1, next_thought_needed=false, current_focus="direct response", key_insights=["No external data needed"], potential_issues=[], confidence_level=0.9)
META_TOOL_CALL: direct_response(final_answer="...")

**Research Task Pattern:**
META_TOOL_CALL: internal_reasoning(thought="I need to gather information from external sources", ...)
META_TOOL_CALL: manage_internal_task_stack(action="add", task_description="Research topic X")
META_TOOL_CALL: delegate_to_llm_tool_node(task_description="Search for information about X", tools_list=["search_web"])
[Wait for result]
META_TOOL_CALL: internal_reasoning(thought="I have the research data, now I can formulate response", ...)
META_TOOL_CALL: direct_response(final_answer="Based on my research: ...")

**Complex Project Pattern:**
META_TOOL_CALL: internal_reasoning(thought="This requires multiple steps with dependencies", ...)
META_TOOL_CALL: create_and_execute_plan(goals=["Step 1: Gather data A", "Step 2: Gather data B", "Step 3: Analyze A and B together", "Step 4: Create final report"])
[Wait for plan completion]
META_TOOL_CALL: direct_response(final_answer="I've completed your complex request...")

## ENHANCED ANTI-LOOP ENFORCEMENT:
- Current Loop: {self.current_loop_count}/{self.max_reasoning_loops}
- Auto-Recovery Attempts: {getattr(self, 'auto_recovery_attempts', 0)}/{getattr(self, 'max_auto_recovery', 3)}
- Last Actions: {', '.join(getattr(self, 'last_action_signatures', [])[-3:]) if hasattr(self, 'last_action_signatures') else 'None'}

**⚠️ LOOP PREVENTION RULES:**
1. If you just read a variable, DO NOT read the same variable again
2. If you completed a task, DO NOT repeat the same work
3. If results exist in unified context, DO NOT recreate them
4. Always advance to next logical step

{self._get_current_step_requirements()}

## YOUR NEXT ACTION (Choose ONE):
Based on your current task, unified context, and available variables, what is your next concrete action?

**DECISION TREE:**
1. ❓ No current task? → Add tasks from outline
2. 📖 Current task needs data? → Check variables and unified context first (read_from_variables)
3. 🔧 Need to execute tools and reason over up to 3 steps? → Use delegate_to_llm_tool_node
4. ✅ Task complete? → Mark complete and advance
5. 🎯 All outline done? → Provide direct_response

Latest unified context: (note delegation results could be wrong or misleading)
<immediate_context>
{immediate_context}
</immediate_context>

must validate <immediate_context> output!
- validate the <immediate_context> output! before proceeding with the outline!
- output compleat fail -> direct_response
- informations missing or output recovery needed -> repeat step with a different strategy
- not enough structure -> use create_and_execute_plan meta-tool call
- output is valid -> continue with the outline!
- if dynamic Planing is needed, you must use the appropriate meta-tool call

**Remember**:
- work step by step max call 3 meta-tool calls in one run.
- only use direct_response if the outline is complete and context from <immediate_context> is enough to answer the query!
- Your job is to work systematically through your outline using your task stack, while leveraging the unified context system to avoid duplicate work and maintain context."""

        return prompt

    def _format_unified_context_for_reasoning(self, unified_context: dict[str, Any]) -> str:
        """Format unified context für reasoning prompt"""
        try:
            context_parts = []

            # Session info
            session_stats = unified_context.get('session_stats', {})
            context_parts.append(
                f"Session: {unified_context.get('session_id', 'unknown')} with {session_stats.get('current_session_length', 0)} messages")

            # Chat history summary
            chat_history = unified_context.get('chat_history', [])
            if chat_history:
                recent_messages = len([msg for msg in chat_history if msg.get('role') == 'user'])
                context_parts.append(f"Conversation: {recent_messages} user queries in current context")

                # Show last user message for reference
                last_user_msg = None
                for msg in reversed(chat_history):
                    if msg.get('role') == 'user':
                        last_user_msg = msg.get('content', '')[:100] + "..."
                        break
                if last_user_msg:
                    context_parts.append(f"Latest user query: {last_user_msg}")

            # Execution state
            execution_state = unified_context.get('execution_state', {})
            active_tasks = execution_state.get('active_tasks', [])
            recent_completions = execution_state.get('recent_completions', [])
            if active_tasks or recent_completions:
                context_parts.append(
                    f"Execution: {len(active_tasks)} active, {len(recent_completions)} completed tasks")

            # Available data
            variables = unified_context.get('variables', {})
            recent_results = variables.get('recent_results', [])
            if recent_results:
                context_parts.append(f"Available Results: {len(recent_results)} recent task results accessible")

            return "\n".join(context_parts)

        except Exception as e:
            return f"Error formatting unified context: {str(e)}"

    def _build_recent_results_from_unified_context(self, unified_context: dict[str, Any]) -> str:
        """Build recent results context from unified context"""
        try:
            variables = unified_context.get('variables', {})
            recent_results = variables.get('recent_results', [])

            if not recent_results:
                return "**No recent results available from unified context**"

            result_context = """**🔍 RECENT RESULTS FROM UNIFIED CONTEXT:**"""

            for i, result in enumerate(recent_results[:3], 1):  # Top 3 results
                task_id = result.get('task_id', f'result_{i}')
                preview = result.get('preview', 'No preview')
                success = result.get('success', False)
                status_icon = "✅" if success else "❌"

                result_context += f"\n{status_icon} {task_id}: {preview}"

            result_context += "\n\n**Quick Access Keys Available:**"
            result_context += "\n- Use read_from_variables(scope='results', key='task_id.data') for specific results"
            result_context += "\n- Check delegation.latest for most recent delegation results"

            return result_context

        except Exception as e:
            return f"**Error accessing recent results: {str(e)}**"

    def _generate_loop_warning(self) -> str:
        """Generate loop warning if repetitive behavior detected"""
        if len(self.last_action_signatures) >= 3:
            recent_actions = self.last_action_signatures[-3:]
            if len(set(recent_actions)) <= 2:
                return """
⚠️ **LOOP WARNING DETECTED** ⚠️
You are repeating similar actions. MUST change approach:
- If you just read variables, act on the results
- If you delegated tasks, check the results
- Complete current task and advance to next step
- DO NOT repeat the same meta-tool calls
    """
        return ""

    def _get_current_stack_task(self) -> dict[str, Any]:
        """Get current pending task from stack for direct visibility"""
        if not self.internal_task_stack:
            return {}

        pending_tasks = [task for task in self.internal_task_stack if task.get("status", "pending") == "pending"]
        if pending_tasks:
            current_task = pending_tasks[0]  # Get first pending task
            return {
                "description": current_task.get("description", ""),
                "outline_step_ref": current_task.get("outline_step_ref", ""),
                "status": current_task.get("status", "pending"),
                "added_at": current_task.get("added_at", ""),
                "task_index": self.internal_task_stack.index(current_task) + 1,
                "total_tasks": len(self.internal_task_stack)
            }

        return {}

    def _analyze_current_task(self, current_task: dict[str, Any]) -> str:
        """Analyze current task and provide guidance"""
        if not current_task:
            return "❌ NO CURRENT TASK - Add tasks from your outline immediately!"

        description = current_task.get("description", "")
        outline_ref = current_task.get("outline_step_ref", "")

        analysis = f"""CURRENT TASK IDENTIFIED:
Task: {description}
Outline Reference: {outline_ref}
Position: {current_task.get('task_index', '?')}/{current_task.get('total_tasks', '?')}

RECOMMENDED ACTION:"""

        # Analyze task content for recommendations
        if "read" in description.lower() or "file" in description.lower():
            analysis += "\n1. Check if file content already exists in variables (read_from_variables)"
            analysis += "\n2. If not found, use delegate_to_llm_tool_node with read_file tool"
        elif "write" in description.lower() or "create" in description.lower():
            analysis += "\n1. Check if content is ready in variables"
            analysis += "\n2. Use delegate_to_llm_tool_node with write_file tool"
        elif "analyze" in description.lower() or "question" in description.lower():
            analysis += "\n1. Read existing data from variables"
            analysis += "\n2. Process the information and provide direct_response"
        else:
            analysis += "\n1. Break down the task into specific actions"
            analysis += "\n2. Verify last Task Delegation results"

        return analysis

    def _get_immediate_context_for_prompt(self) -> str:
        """Get immediate context additions from recent meta-tool executions"""
        recent_results = [
            entry for entry in self.reasoning_context[-5:]  # Last 5 entries
            if entry.get("type") == "meta_tool_result"
        ]

        if not recent_results:
            return "No recent meta-tool results"

        context_parts = ["📊 IMMEDIATE CONTEXT FROM RECENT ACTIONS:"]

        for result in recent_results:
            meta_tool = result.get("meta_tool", "unknown")
            content = result.get("content", "")
            loop = result.get("loop", "?")

            # Format based on meta-tool type
            if meta_tool == "delegate_to_llm_tool_node":
                context_parts.append(f"✅ DELEGATION RESULT (Loop {loop}):")
                context_parts.append(f"   {content}")
            elif meta_tool == "read_from_variables":
                context_parts.append(f"📖 VARIABLE READ (Loop {loop}):")
                context_parts.append(f"   {content}")
            elif meta_tool == "manage_internal_task_stack":
                context_parts.append(f"📋 TASK UPDATE (Loop {loop}):")
                context_parts.append(f"   {content}")
            else:
                context_parts.append(f"🔧 {meta_tool.upper()} (Loop {loop}):")
                context_parts.append(f"   {content}")

        return "\n".join(context_parts)

    def _summarize_reasoning_context(self) -> str:
        """Enhanced reasoning context summary with immediate result visibility"""
        if not self.reasoning_context:
            return "No previous reasoning steps"

        # Separate different types of context entries
        reasoning_entries = []
        meta_tool_results = []
        errors = []

        for entry in self.reasoning_context:
            entry_type = entry.get("type", "unknown")

            if entry_type == "reasoning":
                reasoning_entries.append(entry)
            elif entry_type == "meta_tool_result":
                meta_tool_results.append(entry)
            elif entry_type == "error":
                errors.append(entry)

        summary_parts = []

        # Show recent meta-tool results FIRST for immediate visibility
        if meta_tool_results:
            summary_parts.append("🔍 RECENT RESULTS:")
            for result in meta_tool_results[-3:]:  # Last 3 results
                meta_tool = result.get("meta_tool", "unknown")
                content = result.get("content", "")[:3000] + "..."
                loop = result.get("loop", "?")
                summary_parts.append(f"  [{meta_tool}] Loop {loop}: {content}")

        # Show reasoning summary
        if reasoning_entries:
            summary_parts.append(f"\n💭 REASONING: {len(reasoning_entries)} reasoning cycles completed")

        # Show errors if any
        if errors:
            summary_parts.append(f"\n⚠️ ERRORS: {len(errors)} errors encountered")
            for error in errors[-2:]:  # Last 2 errors
                content = error.get("content", "")[:1500]
                summary_parts.append(f"  Error: {content}")

        return "\n".join(summary_parts)

    def _get_current_step_requirements(self) -> str:
        """Get requirements for current outline step"""
        if not self.outline or not self.outline.get("steps"):
            return "ERROR: No outline available"

        steps = self.outline["steps"]
        if self.current_outline_step >= len(steps):
            return "All outline steps completed - must provide final response"

        current_step = steps[self.current_outline_step]

        requirements = f"""CURRENT STEP FOCUS:
Description: {current_step.get('description', 'Unknown')}
Required Method: {current_step.get('method', 'Unknown')}
Expected Outcome: {current_step.get('expected_outcome', 'Unknown')}
Success Criteria: {current_step.get('success_criteria', 'Unknown')}
Current Status: {current_step.get('status', 'pending')}

You MUST use the specified method and achieve the expected outcome before advancing."""

        return requirements

    def _get_performance_context(self) -> str:
        """Get performance context with accurate metrics"""
        if not self.performance_metrics:
            return "No performance metrics available"

        metrics_parts = []

        # Core metrics
        avg_time = self.performance_metrics.get("avg_loop_time", 0)
        efficiency = self.performance_metrics.get("action_efficiency", 0)
        total_loops = self.performance_metrics.get("total_loops", 0)
        progress_loops = self.performance_metrics.get("progress_loops", 0)

        metrics_parts.append(f"Avg Loop Time: {avg_time:.2f}s")
        metrics_parts.append(f"Progress Rate: {efficiency:.1%}")
        metrics_parts.append(f"Action Efficiency: {efficiency:.1%}")

        # Performance warnings
        if total_loops > 3 and efficiency < 0.5:
            metrics_parts.append("⚠️ LOW EFFICIENCY - Need more progress actions")
        elif total_loops > 5 and efficiency < 0.3:
            metrics_parts.append("🔴 VERY LOW EFFICIENCY - Review approach")

        # Loop detection warning based on actual metrics
        if len(self.last_action_signatures) > 3:
            unique_recent = len(set(self.last_action_signatures[-3:]))
            if unique_recent <= 1:
                metrics_parts.append("⚠️ LOOP PATTERN DETECTED - Change approach required")

        return "; ".join(metrics_parts)

    def _track_action_type(self, action_type: str, success: bool = True):
        """Track specific action types for detailed performance analysis"""
        if not hasattr(self, 'action_tracking'):
            self.action_tracking = {}

        if action_type not in self.action_tracking:
            self.action_tracking[action_type] = {"total": 0, "successful": 0}

        self.action_tracking[action_type]["total"] += 1
        if success:
            self.action_tracking[action_type]["successful"] += 1

        # Update overall action efficiency based on all action types
        total_actions = sum(stats["total"] for stats in self.action_tracking.values())
        successful_actions = sum(stats["successful"] for stats in self.action_tracking.values())

        if total_actions > 0:
            self.performance_metrics["detailed_action_efficiency"] = successful_actions / total_actions


    def _detect_infinite_loop(self) -> bool:
        """Enhanced infinite loop detection with multiple patterns"""
        if len(self.last_action_signatures) < 3:
            return False

        # 1. Immediate repetition (same action 3+ times)
        recent_actions = self.last_action_signatures[-3:]
        if len(set(recent_actions)) == 1:
            return True

        # 2. Pattern repetition (AB-AB-AB pattern)
        if len(self.last_action_signatures) >= 6:
            pattern1 = self.last_action_signatures[-6:-3]
            pattern2 = self.last_action_signatures[-3:]
            if pattern1 == pattern2:
                return True

        # 3. Variable read loops (multiple reads of same variable)
        variable_reads = [sig for sig in self.last_action_signatures if sig.startswith("read_from_variables")]
        if len(variable_reads) >= 3:
            # Extract variable signatures from recent reads
            recent_var_reads = variable_reads[-3:]
            if len(set(recent_var_reads)) <= 2:  # Repeated variable reads
                return True

        # 4. No outline progress for extended loops
        if self.current_loop_count > 5:
            if not hasattr(self, '_last_step_progress_loop'):
                self._last_step_progress_loop = {}

            last_progress = self._last_step_progress_loop.get(self.current_outline_step, 0)
            if self.current_loop_count - last_progress > 4:  # No step progress for 4+ loops
                return True

        # 5. Same task stack state for multiple loops
        if hasattr(self, '_task_stack_states'):
            stack_signature = hash(
                str([(t.get('status'), t.get('description')[:20]) for t in self.internal_task_stack]))
            if stack_signature in self._task_stack_states:
                repetitions = self._task_stack_states[stack_signature]
                if repetitions >= 4:
                    return True
                self._task_stack_states[stack_signature] = repetitions + 1
            else:
                self._task_stack_states[stack_signature] = 1
        else:
            self._task_stack_states = {}

        return False

    async def _trigger_auto_recovery(self, prep_res):
        """Trigger auto-recovery mechanism"""
        self.auto_recovery_attempts += 1

        # Store failure pattern
        if self.variable_manager:
            failure_data = {
                "timestamp": datetime.now().isoformat(),
                "loop_count": self.current_loop_count,
                "outline_step": self.current_outline_step,
                "last_actions": self.last_action_signatures[-5:],
                "recovery_attempt": self.auto_recovery_attempts
            }
            failures = self.variable_manager.get("reasoning.failure_patterns", [])
            failures.append(failure_data)
            self.variable_manager.set("reasoning.failure_patterns", failures[-20:])  # Keep last 20

        # Recovery strategies
        if self.auto_recovery_attempts == 1:
            # Force outline step advancement
            await self._force_outline_advancement(prep_res)
        elif self.auto_recovery_attempts == 2:
            # Skip current step and move to next
            await self._emergency_step_skip(prep_res)
        else:
            # Final emergency: force completion
            await self._emergency_completion(prep_res)

    async def _force_outline_advancement(self, prep_res):
        """Force advancement to next outline step"""
        if self.outline and self.current_outline_step < len(self.outline["steps"]):
            current_step = self.outline["steps"][self.current_outline_step]
            current_step["status"] = "force_completed"
            current_step["completion_method"] = "auto_recovery"

            self.current_outline_step += 1

            # Add to context
            self.reasoning_context.append({
                "type": "auto_recovery",
                "content": f"Force advanced to step {self.current_outline_step + 1} due to loop detection",
                "recovery_attempt": self.auto_recovery_attempts,
                "timestamp": datetime.now().isoformat()
            })

    async def _emergency_step_skip(self, prep_res):
        """Emergency skip of problematic step"""
        if self.outline and self.current_outline_step < len(self.outline["steps"]) - 1:
            current_step = self.outline["steps"][self.current_outline_step]
            current_step["status"] = "emergency_skipped"
            current_step["skip_reason"] = "loop_recovery"

            self.current_outline_step += 1

            # Add to context
            self.reasoning_context.append({
                "type": "emergency_skip",
                "content": f"Emergency skipped step {self.current_outline_step} and advanced to step {self.current_outline_step + 1}",
                "recovery_attempt": self.auto_recovery_attempts,
                "timestamp": datetime.now().isoformat()
            })

    async def _emergency_completion(self, prep_res):
        """Emergency completion of reasoning"""
        # Mark all remaining steps as emergency completed
        if self.outline:
            for i in range(self.current_outline_step, len(self.outline["steps"])):
                self.outline["steps"][i]["status"] = "emergency_completed"

            self.current_outline_step = len(self.outline["steps"])

        # Add to context
        self.reasoning_context.append({
            "type": "emergency_completion",
            "content": "Emergency completion triggered due to excessive recovery attempts",
            "recovery_attempt": self.auto_recovery_attempts,
            "timestamp": datetime.now().isoformat()
        })

    def _update_performance_metrics(self, loop_start_time: float, progress_made: bool):
        """Update performance metrics with accurate action efficiency tracking"""
        loop_duration = time.time() - loop_start_time

        # Initialize metrics if needed
        if not hasattr(self, 'performance_metrics') or not self.performance_metrics:
            self.performance_metrics = {
                "loop_times": [],
                "progress_loops": 0,
                "total_loops": 0
            }

        # Update core metrics
        self.performance_metrics["loop_times"].append(loop_duration)
        self.performance_metrics["total_loops"] += 1

        if progress_made:
            self.performance_metrics["progress_loops"] += 1

        # Calculate derived metrics
        total = self.performance_metrics["total_loops"]
        progress = self.performance_metrics["progress_loops"]

        self.performance_metrics["avg_loop_time"] = sum(self.performance_metrics["loop_times"]) / len(
            self.performance_metrics["loop_times"])
        self.performance_metrics["action_efficiency"] = progress / total if total > 0 else 0.0
        self.performance_metrics["progress_rate"] = self.performance_metrics["action_efficiency"]  # Same metric

        # Keep only recent loop times for memory efficiency
        if len(self.performance_metrics["loop_times"]) > 10:
            self.performance_metrics["loop_times"] = self.performance_metrics["loop_times"][-10:]

    def _add_context_to_reasoning(self, context_addition: str, meta_tool_name: str,
                                  execution_details: dict = None) -> None:
        """Add context addition to reasoning context for immediate visibility in next LLM prompt"""
        if not context_addition:
            return

        # Create structured context entry
        context_entry = {
            "type": "meta_tool_result",
            "content": context_addition,
            "meta_tool": meta_tool_name,
            "loop": self.current_loop_count,
            "outline_step": getattr(self, 'current_outline_step', 0),
            "timestamp": datetime.now().isoformat()
        }

        # Add execution details if provided
        if execution_details:
            context_entry["execution_details"] = {
                "duration": execution_details.get("execution_duration", 0),
                "success": execution_details.get("execution_success", False),
                "tool_category": execution_details.get("tool_category", "unknown")
            }

        # Add to reasoning context for immediate visibility
        self.reasoning_context.append(context_entry)

        # Store in variables for persistent access
        if self.agent_instance:
            if not self.agent_instance.shared.get("system_context"):
                self.agent_instance.shared["system_context"] = {}
            if not self.agent_instance.shared["system_context"].get("reasoning_context"):
                self.agent_instance.shared["system_context"]["reasoning_context"] = {}

            result_key = f"reasoning.loop_{self.current_loop_count}_{meta_tool_name}"
            self.agent_instance.shared["system_context"]["reasoning_context"][result_key] = {
                "context_addition": context_addition,
                "meta_tool": meta_tool_name,
                "timestamp": datetime.now().isoformat(),
                "loop": self.current_loop_count
            }

    async def _parse_and_execute_meta_tools(self, llm_response: str, prep_res: dict) -> dict[str, Any]:
        """Enhanced meta-tool parsing with comprehensive progress tracking"""

        result = {
            "final_result": None,
            "action_taken": None,
            "progress_made": False,
            "context_addition": None
        }

        progress_tracker = prep_res.get("progress_tracker")
        session_id = prep_res.get("session_id")

        # Pattern to match META_TOOL_CALL: tool_name(args...)
        pattern = r'META_TOOL_CALL:'
        matches = _extract_meta_tool_calls(llm_response, pattern)

        if not matches and progress_tracker:
            # No meta-tools found in response
            await progress_tracker.emit_event(ProgressEvent(
                event_type="meta_tool_analysis",
                node_name="LLMReasonerNode",
                session_id=session_id,
                status=NodeStatus.COMPLETED,
                success=True,  # Die Analyse selbst war erfolgreich
                node_phase="analysis_complete",  # Verwendung des dedizierten Feldes
                llm_output=llm_response,  # Speichert die vollständige analysierte Antwort
                metadata={
                    "analysis_result": "no_meta_tools_detected",
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "context_size": len(self.reasoning_context),
                    "performance_warning": len(self.reasoning_context) > 10 and self.current_loop_count > 5
                }
            ))
            result["context_addition"] = "No action taken - this violates outline-driven execution requirements"
            self._add_context_to_reasoning(result["context_addition"], "invalid", {})

            return result

        for i, (tool_name, args_str) in enumerate(matches):
            meta_tool_start = time.perf_counter()

            # Track action signature for loop detection
            action_signature = f"{tool_name}:{hash(args_str) % 1000}"
            self.last_action_signatures.append(action_signature)
            if len(self.last_action_signatures) > 10:
                self.last_action_signatures = self.last_action_signatures[-10:]

            try:
                # Parse arguments with enhanced error handling
                args = _parse_tool_args(args_str)
                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="tool_call",  # Vereinheitlicht auf "tool_call"
                        node_name="LLMReasonerNode",
                        session_id=session_id,
                        status=NodeStatus.RUNNING,
                        tool_name=tool_name,
                        is_meta_tool=True,  # Klares Flag für Meta-Tools
                        tool_args=args,
                        task_id=f"meta_tool_{tool_name}_{i + 1}",
                        metadata={
                            "reasoning_loop": self.current_loop_count,
                            "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0
                        }
                    ))
                rprint(f"Parsed args: {args}")

                # Execute meta-tool with detailed tracking
                meta_result = None
                execution_details = {
                    "meta_tool_name": tool_name,
                    "parsed_args": args,
                    "execution_success": False,
                    "execution_duration": 0.0,
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "context_before_size": len(self.reasoning_context),
                    "task_stack_before_size": len(self.internal_task_stack),
                    "tool_category": self._get_tool_category(tool_name),
                    "execution_phase": "executing"
                }

                if tool_name == "internal_reasoning":
                    meta_result = await self._execute_enhanced_internal_reasoning(args, prep_res)
                    execution_details.update({
                        "thought_number": args.get("thought_number", 1),
                        "total_thoughts": args.get("total_thoughts", 1),
                        "current_focus": args.get("current_focus", ""),
                        "confidence_level": args.get("confidence_level", 0.5),
                        "key_insights": args.get("key_insights", []),
                        "key_insights_count": len(args.get("key_insights", [])),
                        "potential_issues_count": len(args.get("potential_issues", [])),
                        "next_thought_needed": args.get("next_thought_needed", False),
                        "internal_reasoning_log_size": len(getattr(self, 'internal_reasoning_log', [])),
                        "reasoning_depth": self._calculate_reasoning_depth(),
                        "outline_step_progress": args.get("outline_step_progress", "")
                    })
                    result["action_taken"] = False

                elif tool_name == "manage_internal_task_stack":
                    meta_result = await self._execute_enhanced_task_stack(args, prep_res)
                    execution_details.update({
                        "stack_action": args.get("action", "unknown"),
                        "task_description": args.get("task_description", ""),
                        "outline_step_ref": args.get("outline_step_ref", ""),
                        "stack_size_before": len(self.internal_task_stack),
                        "stack_size_after": 0  # Will be updated below
                    })
                    execution_details["stack_size_after"] = len(self.internal_task_stack)
                    execution_details["stack_change"] = execution_details["stack_size_after"] - execution_details[
                        "stack_size_before"]
                    result["action_taken"] = True

                elif tool_name == "delegate_to_llm_tool_node":
                    meta_result = await self._execute_enhanced_delegate_llm_tool(args, prep_res)
                    execution_details.update({
                        "delegated_task_description": args.get("task_description", ""),
                        "tools_list": args.get("tools_list", []),
                        "tools_count": len(args.get("tools_list", [])),
                        "delegation_target": "LLMToolNode",
                        "sub_system_execution": True,
                        "delegation_complexity": self._assess_delegation_complexity(args),
                        "outline_step_completion": args.get("outline_step_completion", False)
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif False and tool_name == "create_and_execute_plan":
                    meta_result = await self._execute_enhanced_create_plan(args, prep_res)
                    execution_details.update({
                        "goals_list": args.get("goals", []),
                        "goals_count": len(args.get("goals", [])),
                        "plan_execution_target": "TaskPlanner_TaskExecutor",
                        "sub_system_execution": True,
                        "complex_workflow": True,
                        "estimated_complexity": self._estimate_plan_complexity(args.get("goals", [])),
                        "outline_step_completion": args.get("outline_step_completion", False)
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif False and tool_name == "create_and_run_micro_plan":
                    meta_result = await self._execute_create_and_run_micro_plan(args, prep_res)
                    execution_details.update({
                        "plan_data": args.get("plan_data", {}),
                        "plan_tasks_count": len(args.get("plan_data", {}).get("tasks", [])),
                        "sub_system_execution": True,
                        "delegation_target": "TaskExecutorNode"
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif tool_name == "advance_outline_step":
                    meta_result = await self._execute_advance_outline_step(args, prep_res)
                    execution_details.update({
                        "step_completed": args.get("step_completed", False),
                        "completion_evidence": args.get("completion_evidence", ""),
                        "next_step_focus": args.get("next_step_focus", ""),
                        "outline_advancement": True,
                        "step_progression": f"{self.current_outline_step}/{len(self.outline.get('steps', [])) if self.outline else 0}"
                    })
                    result["action_taken"] = True
                    result["progress_made"] = True

                elif tool_name == "write_to_variables":
                    meta_result = await self._execute_write_to_variables(args)
                    execution_details.update({
                        "variable_scope": args.get("scope", "reasoning"),
                        "variable_key": args.get("key", ""),
                        "variable_description": args.get("description", ""),
                        "data_persistence": True,
                        "variable_system_operation": "write"
                    })
                    result["action_taken"] = True

                elif tool_name == "read_from_variables":
                    meta_result = await self._execute_read_from_variables(args)
                    execution_details.update({
                        "variable_scope": args.get("scope", "reasoning"),
                        "variable_key": args.get("key", ""),
                        "read_purpose": args.get("purpose", ""),
                        "variable_system_operation": "read",
                        "data_retrieval": True
                    })
                    result["action_taken"] = True

                elif tool_name == "direct_response":

                    final_answer = args.get("final_answer", "Task completed.").replace('\\n', '\n').replace('\\t', '\t')
                    execution_details.update({
                        "final_answer": final_answer,
                        "final_answer_length": len(final_answer),
                        "reasoning_complete": True,
                        "flow_termination": True,
                        "reasoning_summary": self._create_reasoning_summary(),
                        "total_reasoning_steps": len(self.reasoning_context),
                        "outline_completion": True,
                        "steps_completed": args.get("steps_completed", []),
                        "session_completion": True
                    })

                    completion_context = f"✅ REASONING COMPLETE: {final_answer}"
                    self._add_context_to_reasoning(completion_context, tool_name, execution_details)

                    # Store successful completion
                    await self._store_successful_completion(prep_res, final_answer)

                    if progress_tracker:
                        meta_tool_duration = time.perf_counter() - meta_tool_start
                        execution_details["execution_duration"] = meta_tool_duration
                        execution_details["execution_success"] = True

                        await progress_tracker.emit_event(ProgressEvent(
                            event_type="meta_tool_call",
                            timestamp=time.time(),
                            node_name="LLMReasonerNode",
                            status=NodeStatus.COMPLETED,
                            session_id=session_id,
                            task_id=f"meta_tool_{tool_name}_{i + 1}",
                            node_duration=meta_tool_duration,
                            success=True,
                            metadata=execution_details
                        ))

                    result["final_result"] = final_answer
                    result["action_taken"] = True
                    result["progress_made"] = True
                    return result

                # test if tool name is meta_tools_registry if so try to run it
                elif tool_name in self.meta_tools_registry:
                    function = self.meta_tools_registry[tool_name]
                    meta_result = await function(**args)
                    result["action_taken"] = True
                    result["progress_made"] = True
                    execution_details.update({
                        "tool_name": tool_name,
                        "tool_args": args,
                        "tool_result": meta_result
                    })

                # test if tool name is in agent tools if so try to run it
                elif tool_name in self.agent_instance.tool_registry:
                    meta_result = await self.agent_instance.arun_function(tool_name, **args)
                    result["action_taken"] = True
                    result["progress_made"] = True
                    execution_details.update({
                        "tool_name": tool_name,
                        "tool_args": args,
                        "tool_result": meta_result
                    })

                else:
                    execution_details.update({
                        "error_type": "unknown_meta_tool",
                        "error_message": f"Unknown meta-tool: {tool_name}",
                        "execution_success": False,
                        "available_meta_tools": ["internal_reasoning", "manage_internal_task_stack",
                                            "delegate_to_llm_tool_node", "create_and_execute_plan",
                                            "advance_outline_step", "write_to_variables", "read_from_variables",
                                            "direct_response"]
                    })

                    if progress_tracker:
                        meta_tool_duration = time.perf_counter() - meta_tool_start
                        await progress_tracker.emit_event(ProgressEvent(
                            event_type="meta_tool_call",
                            timestamp=time.time(),
                            node_name="LLMReasonerNode",
                            status=NodeStatus.FAILED,
                            session_id=session_id,
                            task_id=f"meta_tool_{tool_name}_{i + 1}",
                            node_duration=meta_tool_duration,
                            success=False,
                            metadata=execution_details
                        ))

                    error_context = f"❌ Unknown meta-tool: {tool_name}"
                    self._add_context_to_reasoning(error_context, tool_name, execution_details)
                    wprint(f"Unknown meta-tool: {tool_name}")
                    continue

                # Update execution details with results
                meta_tool_duration = time.perf_counter() - meta_tool_start
                execution_details.update({
                    "execution_duration": meta_tool_duration,
                    "execution_success": True,
                    "context_after_size": len(self.reasoning_context),
                    "task_stack_after_size": len(self.internal_task_stack),
                    "performance_score": self._calculate_tool_performance_score(meta_tool_duration, tool_name),
                    "execution_phase": "completed"
                })
                self._track_action_type(tool_name, success=True)

                # Add result to context
                if meta_result and meta_result.get("context_addition"):
                    result["context_addition"] = meta_result["context_addition"]
                    execution_details["context_addition_length"] = len(meta_result["context_addition"])

                    self._add_context_to_reasoning(meta_result["context_addition"], tool_name, execution_details)

                # Emit success event
                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="meta_tool_call",
                        timestamp=time.time(),
                        node_name="LLMReasonerNode",
                        status=NodeStatus.COMPLETED,
                        session_id=session_id,
                        task_id=f"meta_tool_{tool_name}_{i + 1}",
                        node_duration=meta_tool_duration,
                        success=True,
                        metadata=execution_details
                    ))

            except Exception as e:
                meta_tool_duration = time.perf_counter() - meta_tool_start
                error_details = {
                    "meta_tool_name": tool_name,
                    "execution_success": False,
                    "execution_duration": meta_tool_duration,
                    "error_type": type(e).__name__,
                    "error_message": str(e),
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "parsed_args": args if 'args' in locals() else None,
                    "raw_args_string": args_str,
                    "execution_phase": "meta_tool_error",
                    "context_size_at_error": len(self.reasoning_context),
                    "task_stack_size_at_error": len(self.internal_task_stack),
                    "tool_category": self._get_tool_category(tool_name),
                    "error_context": self._get_error_context(e),
                    "recovery_recommended": self.auto_recovery_attempts < getattr(self, 'max_auto_recovery', 3)
                }

                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="meta_tool_call",
                        timestamp=time.time(),
                        node_name="LLMReasonerNode",
                        status=NodeStatus.FAILED,
                        session_id=session_id,
                        task_id=f"meta_tool_{tool_name}_{i + 1}",
                        node_duration=meta_tool_duration,
                        success=False,
                        metadata=error_details
                    ))

                eprint(f"Meta-tool execution failed for {tool_name}: {e}")
                result["context_addition"] = f"Error executing {tool_name}: {str(e)}"

                self._add_context_to_reasoning(result["context_addition"], tool_name, execution_details)

        # Final summary event if multiple meta-tools were processed
        if len(matches) > 1 and progress_tracker:
            batch_performance = self._calculate_batch_performance(matches)
            reasoning_progress = self._assess_reasoning_progress()

            await progress_tracker.emit_event(ProgressEvent(
                event_type="meta_tool_batch_complete",
                timestamp=time.time(),
                node_name="LLMReasonerNode",
                status=NodeStatus.COMPLETED,
                session_id=session_id,
                metadata={
                    "total_meta_tools_processed": len(matches),
                    "reasoning_loop": self.current_loop_count,
                    "outline_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                    "batch_execution_complete": True,
                    "final_context_size": len(self.reasoning_context),
                    "final_task_stack_size": len(self.internal_task_stack),
                    "meta_tools_executed": [match[0] for match in matches],
                    "execution_phase": "meta_tool_batch_summary",
                    "batch_performance": batch_performance,
                    "reasoning_progress": reasoning_progress,
                    "progress_made": result["progress_made"],
                    "action_taken": result["action_taken"],
                    "outline_status": {
                        "current_step": self.current_outline_step if hasattr(self, 'current_outline_step') else 0,
                        "total_steps": len(self.outline.get('steps', [])) if self.outline else 0,
                        "completion_ratio": (
                                self.current_outline_step / len(self.outline.get('steps', [1]))) if self.outline else 0
                    },
                    "performance_summary": {
                        "loop_efficiency": self.performance_metrics.get("action_efficiency", 0) if hasattr(self,
                                                                                                           'performance_metrics') else 0,
                        "recovery_attempts": getattr(self, 'auto_recovery_attempts', 0),
                        "context_management_active": len(self.reasoning_context) >= getattr(self,
                                                                                            'context_summary_threshold',
                                                                                            15)
                    }
                }
            ))

        return result

    async def _execute_enhanced_internal_reasoning(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced internal reasoning with outline step tracking"""
        # Standard internal reasoning execution
        result = await self._execute_internal_reasoning(args, prep_res)

        # Enhanced with outline step progress
        outline_step_progress = args.get("outline_step_progress", "")
        if outline_step_progress and result:
            result["context_addition"] += f"\nOutline Step Progress: {outline_step_progress}"

        # Track reasoning depth for current step
        if not hasattr(self, '_step_reasoning_depth'):
            self._step_reasoning_depth = {}

        current_step = self.current_outline_step
        self._step_reasoning_depth[current_step] = self._step_reasoning_depth.get(current_step, 0) + 1

        # Warn if too much reasoning without action
        if self._step_reasoning_depth[current_step] > 3:
            result["context_addition"] += "\n⚠️ WARNING: Too much reasoning without concrete action for current step"

        return result

    async def _execute_enhanced_task_stack(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced task stack management with outline step tracking"""
        # Get outline step reference
        outline_step_ref = args.get("outline_step_ref", f"step_{self.current_outline_step}")

        # Execute standard task stack management
        result = await self._execute_manage_task_stack(args, prep_res)

        # Enhanced with outline step reference
        if result:
            result["context_addition"] += f"\n[Linked to: {outline_step_ref}]"

        return result

    async def _execute_enhanced_delegate_llm_tool(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced delegation with immediate result visibility and guaranteed storage"""
        task_description = args.get("task_description", "")
        tools_list = args.get("tools_list", [])
        outline_step_completion = args.get("outline_step_completion", False)

        # Generate unique delegation ID for this execution
        delegation_id = f"delegation_loop_{self.current_loop_count}"

        # Prepare shared state for LLMToolNode with enhanced result capture
        llm_tool_shared = {
            "current_task_description": task_description,
            "current_query": task_description,
            "formatted_context": {
                "recent_interaction": f"Reasoner delegating task: {task_description}",
                "session_summary": self._get_reasoning_summary(),
                "task_context": f"Loop {self.current_loop_count} delegation - CAPTURE ALL RESULTS"
            },
            "variable_manager": prep_res.get("variable_manager"),
            "agent_instance": prep_res.get("agent_instance"),
            "available_tools": tools_list,
            "tool_capabilities": prep_res.get("tool_capabilities", {}),
            "fast_llm_model": prep_res.get("fast_llm_model"),
            "complex_llm_model": prep_res.get("complex_llm_model"),
            "progress_tracker": prep_res.get("progress_tracker"),
            "session_id": prep_res.get("session_id"),
            "use_fast_response": True
        }

        try:
            # Execute LLMToolNode
            llm_tool_node = LLMToolNode()
            await llm_tool_node.run_async(llm_tool_shared)

            # IMMEDIATE RESULT EXTRACTION - Critical for visibility
            final_response = llm_tool_shared.get("current_response", "No response captured")
            tool_calls_made = llm_tool_shared.get("tool_calls_made", 0)
            tool_results = llm_tool_shared.get("results", {})

            # GUARANTEED STORAGE - Multiple storage patterns for reliability
            delegation_result = {
                "task_description": task_description,
                "tools_used": tools_list,
                "tool_calls_made": tool_calls_made,
                "final_response": final_response,
                "results": tool_results,
                "timestamp": datetime.now().isoformat(),
                "delegation_id": delegation_id,
                "outline_step": self.current_outline_step,
                "reasoning_loop": self.current_loop_count,
                "success": True
            }

            # CRITICAL: Store immediately with multiple access patterns
            if self.variable_manager:
                # 1. Primary delegation storage
                self.variable_manager.set(f"delegation.loop_{self.current_loop_count}", delegation_result)

                # 2. Latest results quick access
                self.variable_manager.set("delegation.latest", delegation_result)

                # 3. Store individual tool results with direct access
                for result_id, result_data in tool_results.items():
                    self.variable_manager.set(f"results.{result_id}.data", result_data.get("data"))

                # 4. Create smart access keys for common patterns
                if "read_file" in tools_list and tool_results:
                    file_content = next((res.get("data") for res in tool_results.values()
                                         if res.get("data") and isinstance(res.get("data"), str)), None)
                    if file_content:
                        self.variable_manager.set("var.file_content", file_content)
                        self.variable_manager.set("latest_file_content", file_content)

                # 5. Update delegation index for discovery
                index = self.variable_manager.get("delegation.index", [])
                index.append({
                    "loop": self.current_loop_count,
                    "task": task_description[:100],
                    "tools": tools_list,
                    "timestamp": datetime.now().isoformat(),
                    "results_available": len(tool_results) > 0
                })
                self.variable_manager.set("delegation.index", index[-20:])

            # Create comprehensive context addition with IMMEDIATE VISIBILITY
            context_addition = f"""DELEGATION COMPLETED (Loop {self.current_loop_count}):
Task: {task_description}
Tools: {', '.join(tools_list)}
Calls Made: {tool_calls_made}
Results Captured: {len(tool_results)} items

FINAL RESULT: {final_response}

- reference variable: delegation.loop_{self.current_loop_count}
DELEGATION END
"""

            # Mark outline step completion if specified
            if outline_step_completion:
                await self._mark_step_completion(prep_res, "delegation_complete", context_addition)

            return {"context_addition": context_addition}

        except Exception as e:
            error_msg = f"❌ DELEGATION FAILED: {str(e)}"
            # Store error for debugging
            if self.variable_manager:
                error_data = {
                    "task": task_description,
                    "error": str(e),
                    "timestamp": datetime.now().isoformat(),
                    "loop": self.current_loop_count
                }
                self.variable_manager.set(f"delegation.error.loop_{self.current_loop_count}", error_data)

            return {"context_addition": error_msg}

    async def _execute_enhanced_create_plan(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Enhanced plan creation with outline step completion tracking"""
        # Check if this completes the outline step
        outline_step_completion = args.get("outline_step_completion", False)

        # Execute standard plan creation
        result = await self._execute_create_plan(args, prep_res)

        # Enhanced with step completion tracking
        if outline_step_completion and result:
            await self._mark_step_completion(prep_res, "create_and_execute_plan", result["context_addition"])
            result["context_addition"] += f"\n✓ OUTLINE STEP {self.current_outline_step + 1} COMPLETED"

        return result

    # Fügen Sie dies innerhalb der Klasse LLMReasonerNode hinzu

    async def _execute_create_and_run_micro_plan(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """
        Erstellt einen kleinen, dynamischen TaskPlan aus den LLM-Daten und führt ihn sofort
        mit dem TaskExecutorNode aus.
        """
        plan_data = args.get("plan_data", {})

        # Validierung der Eingabe
        if not isinstance(plan_data, dict) or "tasks" not in plan_data:
            error_msg = "❌ Micro-Plan-Fehler: Ungültiges `plan_data`-Format. Es muss ein Dictionary mit einem 'tasks'-Schlüssel sein."
            eprint(error_msg)
            return {"context_addition": error_msg}

        try:
            # 1. Task-Objekte aus den Rohdaten erstellen
            tasks = []
            for task_data in plan_data.get("tasks", []):
                task_type = task_data.pop("type", "generic")  # 'type' entfernen, da es kein Task-Argument ist
                task_class = {"LLMTask": LLMTask, "ToolTask": ToolTask, "DecisionTask": DecisionTask}.get(task_type,
                                                                                                          Task)
                tasks.append(task_class(**task_data))

            # 2. TaskPlan-Objekt erstellen
            plan = TaskPlan(
                id=f"micro_plan_{str(uuid.uuid4())[:8]}",
                name=plan_data.get("plan_name", "Dynamischer Micro-Plan"),
                description=plan_data.get("description", "Vom LLMReasoner on-the-fly erstellter Plan"),
                tasks=tasks,
                execution_strategy=plan_data.get("execution_strategy", "sequential")
            )

            # 3. Den TaskExecutorNode vorbereiten und ausführen
            task_executor_instance = prep_res.get("task_executor")
            if not task_executor_instance:
                return {"context_addition": "❌ Micro-Plan-Fehler: TaskExecutorNode-Instanz nicht gefunden."}

            # Shared-State für den Executor-Lauf vorbereiten
            executor_shared = {
                "current_plan": plan,
                "tasks": {task.id: task for task in tasks},
                "variable_manager": self.variable_manager,
                "agent_instance": self.agent_instance,
                "progress_tracker": prep_res.get("progress_tracker"),
                "fast_llm_model": prep_res.get("fast_llm_model"),
                "complex_llm_model": prep_res.get("complex_llm_model"),
                "available_tools": prep_res.get("available_tools", []),
            }

            # 4. Ausführungsschleife für den Executor
            max_cycles = 10
            for i in range(max_cycles):
                result_status = await task_executor_instance.run_async(executor_shared)
                if result_status in ["plan_completed", "execution_error", "needs_dynamic_replan"]:
                    break

            # 5. Ergebnisse zusammenfassen für den Reasoner-Kontext
            final_results = executor_shared.get("results", {})
            completed_tasks = [t for t in tasks if t.status == "completed"]
            failed_tasks = [t for t in tasks if t.status == "failed"]

            summary = f"""✅ Micro-Plan ausgeführt:
    - Plan: '{plan.name}'
    - Status: {len(completed_tasks)} erfolgreich, {len(failed_tasks)} fehlgeschlagen.
    - Ergebnisse sind jetzt in den `results`-Variablen verfügbar (z.B. `{{{{ results.{tasks[0].id}.data }}}}`)."""

            return {"context_addition": summary}

        except Exception as e:
            import traceback
            eprint(f"Fehler bei der Ausführung des Micro-Plans: {e}")
            print(traceback.format_exc())
            return {"context_addition": f"❌ Micro-Plan-Fehler: {str(e)}"}

    async def _execute_advance_outline_step(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute outline step advancement"""
        step_completed = args.get("step_completed", False)
        completion_evidence = args.get("completion_evidence", "")
        next_step_focus = args.get("next_step_focus", "")

        if not self.outline or not self.outline.get("steps"):
            return {"context_addition": "Cannot advance: No outline available"}

        steps = self.outline["steps"]

        if self.current_outline_step >= len(steps):
            return {"context_addition": "Cannot advance: Already at final step"}

        if step_completed:
            # Mark current step as completed
            if self.current_outline_step < len(steps):
                current_step = steps[self.current_outline_step]
                current_step["status"] = "completed"
                current_step["completion_evidence"] = completion_evidence
                current_step["completed_at"] = datetime.now().isoformat()

            # Advance to next step
            self.current_outline_step += 1

            # Store advancement in variables
            if self.variable_manager:
                advancement_data = {
                    "step_completed": self.current_outline_step,
                    "completion_evidence": completion_evidence,
                    "next_step_focus": next_step_focus,
                    "timestamp": datetime.now().isoformat()
                }
                self.variable_manager.set(f"reasoning.step_completions.{self.current_outline_step - 1}",
                                          advancement_data)

            context_addition = f"""✓ STEP {self.current_outline_step} COMPLETED
Evidence: {completion_evidence}
Advanced to Step {self.current_outline_step + 1}/{len(steps)}"""

            if next_step_focus:
                context_addition += f"\nNext Step Focus: {next_step_focus}"

            if self.current_outline_step >= len(steps):
                context_addition += "\n🎯 ALL OUTLINE STEPS COMPLETED - Ready for direct_response"

        else:
            context_addition = f"Step {self.current_outline_step + 1} not yet completed - continue working on current step"

        return {"context_addition": context_addition}

    async def _execute_read_from_variables(self, args: dict) -> dict[str, Any]:
        """Enhanced variable reading with intelligent discovery and loop prevention"""
        if not self.variable_manager:
            return {"context_addition": "❌ Variable system not available"}

        scope = args.get("scope", args.get("query", "reasoning"))
        key = args.get("key", "")
        purpose = args.get("purpose", "")

        # CRITICAL: Check for repeated reads - prevent infinite loops
        read_signature = f"{scope}.{key}"
        if not hasattr(self, '_variable_read_history'):
            self._variable_read_history = []

        # Prevent reading same variable multiple times in short succession
        recent_reads = [r for r in self._variable_read_history if r['signature'] == read_signature]
        if len(recent_reads) >= 2:
            self._variable_read_history.append({
                'signature': read_signature,
                'timestamp': time.time(),
                'loop': self.current_loop_count
            })
            return {
                "context_addition": f"⚠️ LOOP PREVENTION: Already read {read_signature} {len(recent_reads)} times. Try different approach or advance to next task."
            }

        # Record this read attempt
        self._variable_read_history.append({
            'signature': read_signature,
            'timestamp': time.time(),
            'loop': self.current_loop_count
        })

        # Clean old read history (keep last 10)
        if len(self._variable_read_history) > 10:
            self._variable_read_history = self._variable_read_history[-10:]

        if not key:
            return {"context_addition": "❌ Cannot read: No key provided"}

        try:
            # Smart key resolution for common patterns
            resolved_key = self._resolve_smart_key(scope, key)

            # Try direct access first
            value = self.variable_manager.get(resolved_key)

            if value is not None:
                # Format value for display
                value_display = self._format_variable_value(value)

                context_addition = f"""{resolved_key}={value_display}
Access: Successfully retrieved from variable system"""

                return {"context_addition": context_addition}

            else:
                # Enhanced discovery when not found
                discovery_result = self._perform_smart_variable_discovery(scope, key, purpose)
                return {"context_addition": discovery_result}

        except Exception as e:
            return {"context_addition": f"❌ Variable read error: {str(e)}"}

    def _resolve_smart_key(self, scope: str, key: str) -> str:
        """Resolve smart key patterns for common access cases"""
        # Handle delegation results specially
        if scope == "delegation" and "loop_" in key:
            return f"delegation.{key}"
        elif scope == "results" and key.endswith(".data"):
            return f"results.{key}"
        elif scope == "var" or key.startswith("var."):
            return key if key.startswith("var.") else f"var.{key}"
        else:
            return f"{scope}.{key}" if scope != "reasoning" else f"reasoning.{key}"

    def _format_variable_value(self, value: any) -> str:
        """Format variable value for display with intelligent truncation"""
        if isinstance(value, dict | list):
            value_str = json.dumps(value, default=str, indent=2)
        else:
            value_str = str(value)

        # Smart truncation based on content type
        if len(value_str) > 200000:
            if isinstance(value, dict) and "results" in str(value):
                # For result dicts, show structure
                return f"RESULTS DICT ({len(value)} keys):\n" + value_str[:150000] + "\n... [TRUNCATED]"
            elif isinstance(value, str) and (value.startswith("# ") or "markdown" in value.lower()):
                # For file content, show beginning
                return f"FILE CONTENT ({len(value_str)} chars):\n" + value_str[:100000] + "\n... [FULL CONTENT AVAILABLE]"
            else:
                return value_str[:100000] + f"\n... [TRUNCATED - {len(value_str)} total chars]"

        return value_str

    def _perform_smart_variable_discovery(self, scope: str, key: str, purpose: str) -> str:
        """Perform intelligent variable discovery when key not found"""
        # Check latest delegation results first
        latest = self.variable_manager.get("delegation.latest")
        if latest:
            discovery_msg = f"❌ Variable not found: {scope}.{key}\n\n✨ LATEST DELEGATION RESULTS AVAILABLE:"
            discovery_msg += f"\nTask: {latest.get('task_description', 'Unknown')[:100]}"
            discovery_msg += f"\nResults: {len(latest.get('results', {}))} items available"
            discovery_msg += "\nAccess with: delegation.latest"

            # Show actual keys available
            if latest.get('results'):
                discovery_msg += "\n\n🔍 Available result keys:"
                for result_id in latest['results']:
                    discovery_msg += f"\n• results.{result_id}.data"

            return discovery_msg

        # Check delegation index for recent activity
        index = self.variable_manager.get("delegation.index", [])
        if index:
            recent = index[-3:]  # Last 3 delegations
            discovery_msg = f"❌ Variable not found: {scope}.{key}\n\n📚 RECENT DELEGATIONS:"
            for entry in recent:
                discovery_msg += f"\n• Loop {entry['loop']}: {entry['task'][:50]}..."
                discovery_msg += f"  Access: delegation.loop_{entry['loop']}"
            return discovery_msg

        # Fallback: show available scopes
        available_vars = self.variable_manager.get_available_variables()
        return f"❌ Variable not found: {scope}.{key}\n\n📋 Available scopes: {', '.join(available_vars.keys())}"


    async def _execute_write_to_variables(self, args: dict) -> dict[str, Any]:
        """Enhanced variable writing with automatic result storage"""
        if not self.variable_manager:
            return {"context_addition": "❌ Variable system not available"}

        scope = args.get("scope", "reasoning")
        key = args.get("key", "")
        value = args.get("value", "")
        description = args.get("description", "")

        if not key:
            return {"context_addition": "❌ Cannot write to variables: No key provided"}

        try:
            # Create scoped key
            full_key = f"{scope}.{key}" if scope != "reasoning" else f"reasoning.{key}"

            # Write to variables
            self.variable_manager.set(full_key, value)

            # Store enhanced metadata
            metadata = {
                "description": description,
                "written_at": datetime.now().isoformat(),
                "outline_step": getattr(self, 'current_outline_step', 0),
                "reasoning_loop": self.current_loop_count,
                "value_type": type(value).__name__,
                "value_size": len(str(value)) if value else 0,
                "auto_stored": False  # Manual storage
            }
            self.variable_manager.set(f"{full_key}_metadata", metadata)

            # Update storage index for easy discovery
            storage_index = self.variable_manager.get("reasoning.storage_index", [])
            storage_entry = {
                "key": full_key,
                "description": description,
                "timestamp": datetime.now().isoformat(),
                "loop": self.current_loop_count
            }
            storage_index.append(storage_entry)
            self.variable_manager.set("reasoning.storage_index", storage_index[-20:])  # Keep last 20

            context_addition = f"✅ Stored in variables: {full_key}"
            if description:
                context_addition += f"\n📄 Description: {description}"

            # Show how to access it
            context_addition += f"\n🔍 Access with: read_from_variables(scope=\"{scope}\", key=\"{key}\", purpose=\"...\")"

            return {"context_addition": context_addition}

        except Exception as e:
            return {"context_addition": f"❌ Failed to write to variables: {str(e)}"}

    def _auto_store_delegation_results(self, delegation_result: dict, task_description: str) -> str:
        """Automatically store delegation results with smart naming and comprehensive indexing"""
        if not self.variable_manager:
            return "\n❌ Variable system not available for auto-storage"

        storage_summary = []

        try:
            # Store main delegation result with loop reference
            main_key = f"delegation.loop_{self.current_loop_count}"
            self.variable_manager.set(main_key, delegation_result)
            storage_summary.append(f"• {main_key}")

            # Store individual tool results with smart naming
            results = delegation_result.get("results", {})
            smart_keys_created = []

            for result_id, result_data in results.items():
                # Smart naming based on task content and result type
                smart_key = self._generate_smart_key(task_description, result_id, result_data)

                # Store full result
                self.variable_manager.set(smart_key, result_data)
                storage_summary.append(f"• {smart_key}")
                smart_keys_created.append(smart_key)

                # Store data separately for direct access
                if result_data.get("data"):
                    data_key = f"{smart_key}.data"
                    self.variable_manager.set(data_key, result_data["data"])
                    storage_summary.append(f"• {data_key} (direct access)")

                    # Store with generic access pattern
                    generic_data_key = f"results.{result_id}.data"
                    self.variable_manager.set(generic_data_key, result_data["data"])
                    storage_summary.append(f"• {generic_data_key} (standard access)")

            # Update comprehensive quick access index
            quick_access = {
                "latest_delegation": main_key,
                "latest_task": task_description,
                "timestamp": datetime.now().isoformat(),
                "loop": self.current_loop_count,
                "outline_step": getattr(self, 'current_outline_step', 0),
                "stored_keys": [item.replace("• ", "") for item in storage_summary],
                "smart_keys": smart_keys_created,
                "access_patterns": {
                    "main_result": main_key,
                    "by_loop": f"delegation.loop_{self.current_loop_count}",
                    "latest": "reasoning.latest_results",
                    "data_direct": [key for key in storage_summary if ".data" in key]
                }
            }
            self.variable_manager.set("reasoning.latest_results", quick_access)

            # Update global delegation index for easy discovery
            delegation_index = self.variable_manager.get("delegation.index", [])
            index_entry = {
                "loop": self.current_loop_count,
                "task": task_description[:100] + ("..." if len(task_description) > 100 else ""),
                "keys_created": len(storage_summary),
                "timestamp": datetime.now().isoformat(),
                "main_key": main_key,
                "smart_keys": smart_keys_created
            }
            delegation_index.append(index_entry)
            self.variable_manager.set("delegation.index", delegation_index[-50:])  # Keep last 50

            # Store task-specific quick access
            task_hash = hash(task_description) % 10000
            self.variable_manager.set(f"delegation.by_task.{task_hash}", {
                "task_description": task_description,
                "results": quick_access,
                "created_at": datetime.now().isoformat()
            })

            return f"\n📊 Auto-stored results ({len(storage_summary)} entries):\n" + "\n".join(storage_summary[:8]) + (
                f"\n... +{len(storage_summary) - 8} more" if len(storage_summary) > 8 else "")

        except Exception as e:
            return f"\n❌ Auto-storage failed: {str(e)}"

    def _generate_smart_key(self, task_description: str, result_id: str, result_data: dict) -> str:
        """Generate intelligent storage keys based on task content and result type"""
        task_lower = task_description.lower()

        # Analyze task type
        if "read" in task_lower and "file" in task_lower:
            prefix = "file_content"
        elif "write" in task_lower and "file" in task_lower:
            prefix = "file_written"
        elif "create" in task_lower and "file" in task_lower:
            prefix = "file_created"
        elif "search" in task_lower or "find" in task_lower:
            prefix = "search_results"
        elif "analyze" in task_lower or "analysis" in task_lower:
            prefix = "analysis_results"
        elif "list" in task_lower or "directory" in task_lower:
            prefix = "directory_listing"
        elif "download" in task_lower or "fetch" in task_lower:
            prefix = "downloaded_content"
        else:
            # Analyze result data for hints
            result_str = str(result_data).lower()
            if "file" in result_str and "content" in result_str:
                prefix = "file_content"
            elif "search" in result_str or "results" in result_str:
                prefix = "search_results"
            elif "data" in result_str:
                prefix = "task_data"
            else:
                prefix = "task_result"

        # Create unique key with loop and result ID
        return f"{prefix}.loop_{self.current_loop_count}_{result_id}"

    async def _mark_step_completion(self, prep_res: dict, method: str, evidence: str):
        """Mark current outline step as completed"""
        if not self.outline or not self.outline.get("steps"):
            return

        steps = self.outline["steps"]
        if self.current_outline_step < len(steps):
            current_step = steps[self.current_outline_step]
            current_step["status"] = "completed"
            current_step["completion_method"] = method
            current_step["completion_evidence"] = evidence
            current_step["completed_at"] = datetime.now().isoformat()

            # Store in variables
            if self.variable_manager:
                completion_data = {
                    "step_number": self.current_outline_step,
                    "description": current_step.get("description", ""),
                    "method": method,
                    "evidence": evidence,
                    "timestamp": datetime.now().isoformat()
                }
                self.variable_manager.set(f"reasoning.step_completions.{self.current_outline_step}", completion_data)

    async def _store_successful_completion(self, prep_res: dict, final_answer: str):
        """Store successful completion data for future learning"""
        if not self.variable_manager:
            return

        success_data = {
            "query": prep_res["original_query"],
            "final_answer": final_answer,
            "reasoning_loops": self.current_loop_count,
            "outline": self.outline,
            "performance_metrics": self.performance_metrics,
            "auto_recovery_attempts": self.auto_recovery_attempts,
            "completion_timestamp": datetime.now().isoformat(),
            "session_id": prep_res.get("session_id", "default")
        }

        # Store in successful patterns
        successes = self.variable_manager.get("reasoning.successful_patterns", [])
        successes.append(success_data)
        self.variable_manager.set("reasoning.successful_patterns", successes[-20:])  # Keep last 20

        # Update performance statistics
        self._update_success_statistics()

    def _update_success_statistics(self):
        """Update success statistics in variables"""
        if not self.variable_manager:
            return

        # Get current stats
        current_stats = self.variable_manager.get("reasoning.performance.statistics", {})

        # Update stats
        current_stats["total_successful_sessions"] = current_stats.get("total_successful_sessions", 0) + 1
        current_stats["avg_loops_per_success"] = current_stats.get("avg_loops_per_success", 0)

        # Calculate new average
        total_sessions = current_stats["total_successful_sessions"]
        old_avg = current_stats["avg_loops_per_success"] * (total_sessions - 1)
        current_stats["avg_loops_per_success"] = (old_avg + self.current_loop_count) / total_sessions

        # Store updated stats
        self.variable_manager.set("reasoning.performance.statistics", current_stats)

    async def _create_outline_completion_response(self, prep_res: dict) -> str:
        """Create response when outline is completed"""
        if not self.outline:
            return "Outline completion response requested but no outline available"

        steps = self.outline.get("steps", [])
        completed_steps = [s for s in steps if
                           s.get("status") in ["completed", "force_completed", "emergency_completed"]]

        response_parts = []
        response_parts.append("I have completed the structured approach outlined for your request:")

        # Summarize completed steps
        for i, step in enumerate(completed_steps):
            status_indicator = "✓" if step.get("status") == "completed" else "⚠️"
            response_parts.append(f"{status_indicator} Step {i + 1}: {step.get('description', 'Unknown step')}")

            # Add evidence if available
            evidence = step.get("completion_evidence", "")
            if evidence and len(evidence) < 200:
                response_parts.append(f"   Result: {evidence}")

        # Get final results from variables if available
        if self.variable_manager:
            final_results = self.variable_manager.get("reasoning.final_results", {})
            if final_results:
                response_parts.append("\nKey findings:")
                for key, value in final_results.items():
                    if isinstance(value, str) and len(value) < 300:
                        response_parts.append(f"- {key}: {value}")

        response_parts.append(
            f"\nCompleted in {self.current_loop_count} reasoning cycles using outline-driven execution.")

        return "\n".join(response_parts)

    async def _create_enhanced_timeout_response(self, query: str, prep_res: dict) -> str:
        """Create enhanced timeout response with comprehensive progress summary"""
        response_parts = []
        response_parts.append(
            f"I reached my reasoning limit of {self.max_reasoning_loops} steps while working on: {query}")

        # Outline progress
        if self.outline:
            steps = self.outline.get("steps", [])
            completed_steps = [s for s in steps if
                               s.get("status") in ["completed", "force_completed", "emergency_completed"]]
            unfinished_steps = [s for s in steps if s not in completed_steps]

            response_parts.append(f"\nOutline Progress: {len(completed_steps)}/{len(steps)} steps completed")

            if completed_steps:
                response_parts.append("Completed steps:")
                for i, step in enumerate(completed_steps):
                    response_parts.append(f"✓ {step.get('description', f'Step {i + 1}')}")

            if unfinished_steps:
                response_parts.append("Unfinished steps:")
                for i, step in enumerate(unfinished_steps):
                    response_parts.append(f"✗ {step.get('description', f'Step {i + 1}')}")

        # Task stack progress
        if self.internal_task_stack:
            completed_tasks = [t for t in self.internal_task_stack if t.get("status") == "completed"]
            pending_tasks = [t for t in self.internal_task_stack if t.get("status") == "pending"]

            response_parts.append(f"\nTask Progress: {len(completed_tasks)} completed, {len(pending_tasks)} pending")

        # Performance metrics
        if self.performance_metrics:
            response_parts.append(
                f"\nPerformance: {self.performance_metrics.get('action_efficiency', 0):.1%} efficiency, {self.auto_recovery_attempts} recovery attempts")

        # Available results from variables
        if self.variable_manager:
            reasoning_results = self.variable_manager.get("reasoning", {})
            if reasoning_results:
                response_parts.append(f"\nStored findings: {len(reasoning_results)} entries in reasoning variables")

        return "\n".join(response_parts)

    async def _finalize_reasoning_session(self, prep_res: dict, final_result: str):
        """Finalize reasoning session with comprehensive data storage"""
        if not self.variable_manager:
            return

        # Store session completion data
        session_data = {
            "query": prep_res["original_query"],
            "final_result": final_result,
            "reasoning_loops": self.current_loop_count,
            "outline_completion": self.current_outline_step,
            "performance_metrics": self.performance_metrics,
            "auto_recovery_attempts": self.auto_recovery_attempts,
            "context_summaries": len([c for c in self.reasoning_context if c.get("type") == "context_summary"]),
            "completion_timestamp": datetime.now().isoformat(),
            "session_duration": time.time() - time.mktime(datetime.now().timetuple()),
            "success": True
        }

        # Store in session history
        session_history = self.variable_manager.get("reasoning.session_history", [])
        session_history.append(session_data)
        self.variable_manager.set("reasoning.session_history", session_history[-50:])  # Keep last 50 sessions

        # Store outline pattern for reuse
        if self.outline:
            outline_pattern = {
                "query_type": self._classify_query_type(prep_res["original_query"]),
                "outline": self.outline,
                "success": True,
                "loops_used": self.current_loop_count,
                "timestamp": datetime.now().isoformat()
            }
            patterns = self.variable_manager.get("reasoning.successful_patterns.outlines", [])
            patterns.append(outline_pattern)
            self.variable_manager.set("reasoning.successful_patterns.outlines", patterns[-10:])

    def _classify_query_type(self, query: str) -> str:
        """Classify query type for pattern matching"""
        query_lower = query.lower()

        if any(word in query_lower for word in ["search", "find", "look up", "research"]):
            return "research"
        elif any(word in query_lower for word in ["analyze", "compare", "evaluate"]):
            return "analysis"
        elif any(word in query_lower for word in ["create", "generate", "write", "build"]):
            return "creation"
        elif any(word in query_lower for word in ["plan", "strategy", "approach"]):
            return "planning"
        else:
            return "general"

    async def _handle_reasoning_error(self, error: Exception, prep_res: dict, progress_tracker):
        """Enhanced error handling with auto-recovery"""
        eprint(f"Reasoning loop {self.current_loop_count} failed: {error}")

        # Store error in context
        self.reasoning_context.append({
            "type": "error",
            "content": f"Error in loop {self.current_loop_count}: {str(error)}",
            "error_type": type(error).__name__,
            "outline_step": self.current_outline_step,
            "timestamp": datetime.now().isoformat()
        })

        # Store in variables for learning
        if self.variable_manager:
            error_data = {
                "error": str(error),
                "error_type": type(error).__name__,
                "loop": self.current_loop_count,
                "outline_step": self.current_outline_step,
                "timestamp": datetime.now().isoformat(),
                "query": prep_res["original_query"]
            }
            errors = self.variable_manager.get("reasoning.error_log", [])
            errors.append(error_data)
            self.variable_manager.set("reasoning.error_log", errors[-100:])  # Keep last 100 errors

        # Trigger auto-recovery if not already in recovery
        if self.auto_recovery_attempts < self.max_auto_recovery:
            await self._trigger_auto_recovery(prep_res)

    # Keep all existing helper methods like _execute_internal_reasoning, etc.
    # but update them to use the enhanced variable system...

    async def post_async(self, shared, prep_res, exec_res):
        """Enhanced post-processing with comprehensive data storage"""
        final_result = exec_res.get("final_result", "Task processing incomplete")

        # Store comprehensive reasoning artifacts
        shared["reasoning_artifacts"] = {
            "reasoning_loops": exec_res.get("reasoning_loops", 0),
            "reasoning_context": exec_res.get("reasoning_context", []),
            "internal_task_stack": exec_res.get("internal_task_stack", []),
            "outline": exec_res.get("outline"),
            "outline_completion": exec_res.get("outline_completion", 0),
            "performance_metrics": exec_res.get("performance_metrics", {}),
            "auto_recovery_attempts": exec_res.get("auto_recovery_attempts", 0)
        }

        # Enhanced variable system updates
        if self.variable_manager:
            # Store final session results
            final_session_data = {
                "final_result": final_result,
                "completion_timestamp": datetime.now().isoformat(),
                "total_loops": exec_res.get("reasoning_loops", 0),
                "session_success": final_result != "Task processing incomplete",
                "outline_driven_execution": True
            }
            self.variable_manager.set("reasoning.current_session.final_data", final_session_data)

            # Update global performance statistics
            self._update_global_performance_stats(exec_res)

        # Set enhanced response data
        shared["llm_reasoner_result"] = final_result
        shared["current_response"] = final_result

        # Provide enhanced synthesis metadata
        shared["synthesized_response"] = {
            "synthesized_response": final_result,
            "confidence": self._calculate_confidence(exec_res),
            "metadata": {
                "synthesis_method": "outline_driven_reasoner",
                "reasoning_loops": exec_res.get("reasoning_loops", 0),
                "outline_completion": exec_res.get("outline_completion", 0),
                "performance_score": self._calculate_performance_score(exec_res),
                "auto_recovery_used": exec_res.get("auto_recovery_attempts", 0) > 0
            }
        }

        return "reasoner_complete"

    def _update_global_performance_stats(self, exec_res: dict):
        """Update global performance statistics in variables"""
        if not self.variable_manager:
            return

        stats = self.variable_manager.get("reasoning.global_performance", {})

        # Update counters
        stats["total_sessions"] = stats.get("total_sessions", 0) + 1
        stats["total_loops"] = stats.get("total_loops", 0) + exec_res.get("reasoning_loops", 0)
        stats["total_recoveries"] = stats.get("total_recoveries", 0) + exec_res.get("auto_recovery_attempts", 0)

        # Calculate averages
        stats["avg_loops_per_session"] = stats["total_loops"] / stats["total_sessions"]
        stats["recovery_rate"] = stats["total_recoveries"] / stats["total_sessions"]

        # Success tracking
        if exec_res.get("final_result") != "Task processing incomplete":
            stats["successful_sessions"] = stats.get("successful_sessions", 0) + 1
            stats["success_rate"] = stats["successful_sessions"] / stats["total_sessions"]

        self.variable_manager.set("reasoning.global_performance", stats)

    def _calculate_confidence(self, exec_res: dict) -> float:
        """Calculate confidence score based on execution results"""
        base_confidence = 0.5

        # Outline completion boosts confidence
        outline = exec_res.get("outline")
        if outline:
            completion_ratio = exec_res.get("outline_completion", 0) / len(outline.get("steps", [1]))
            base_confidence += 0.3 * completion_ratio

        # Low recovery attempts boost confidence
        recovery_attempts = exec_res.get("auto_recovery_attempts", 0)
        if recovery_attempts == 0:
            base_confidence += 0.15
        elif recovery_attempts == 1:
            base_confidence += 0.05

        # Reasonable loop count boosts confidence
        loops = exec_res.get("reasoning_loops", 0)
        if 3 <= loops <= 15:
            base_confidence += 0.1

        # Performance metrics
        performance = exec_res.get("performance_metrics", {})
        if performance.get("action_efficiency", 0) > 0.7:
            base_confidence += 0.1

        return min(1.0, max(0.0, base_confidence))

    def _calculate_performance_score(self, exec_res: dict) -> float:
        """Calculate overall performance score"""
        score = 0.5

        # Efficiency score
        performance = exec_res.get("performance_metrics", {})
        action_efficiency = performance.get("action_efficiency", 0)
        score += 0.3 * action_efficiency

        # Completion score
        outline = exec_res.get("outline")
        if outline:
            completion_ratio = exec_res.get("outline_completion", 0) / len(outline.get("steps", [1]))
            score += 0.4 * completion_ratio

        # Recovery penalty
        recovery_attempts = exec_res.get("auto_recovery_attempts", 0)
        score -= 0.1 * recovery_attempts

        return min(1.0, max(0.0, score))


    def _summarize_reasoning_context(self) -> str:
        """Summarize the current reasoning context"""
        if not self.reasoning_context:
            return "No previous reasoning steps"

        summary_parts = []
        for entry in self.reasoning_context[-5:]:  # Last 5 entries
            entry_type = entry.get("type", "unknown")
            content = entry.get("content", "")

            if entry_type == "reasoning":
                # Truncate long reasoning content
                content_preview = content[:20000] + "..." if len(content) > 20000 else content
                summary_parts.append(f"Loop {entry.get('loop', '?')}: {content_preview}")
            elif entry_type == "meta_tool_result":
                summary_parts.append(f"Result: {content[:150]}...")
            elif entry_type == "error":
                summary_parts.append(f"Error: {content}")

        return "\n".join(summary_parts)

    def _summarize_task_stack(self) -> str:
        """Summarize the internal task stack"""
        if not self.internal_task_stack:
            return "No tasks in stack"

        summary_parts = []
        for i, task in enumerate(self.internal_task_stack):
            status = task.get("status", "pending")
            description = task.get("description", "No description")
            summary_parts.append(f"{i + 1}. [{status.upper()}] {description}")

        return "\n".join(summary_parts)

    def _get_tool_category(self, tool_name: str) -> str:
        """Get category for meta-tool"""
        categories = {
            "internal_reasoning": "thinking",
            "manage_internal_task_stack": "planning",
            "delegate_to_llm_tool_node": "delegation",
            "create_and_execute_plan": "orchestration",
            "direct_response": "completion"
        }
        return categories.get(tool_name, "unknown")

    def _calculate_reasoning_depth(self) -> int:
        """Calculate current reasoning depth"""
        reasoning_entries = [entry for entry in self.reasoning_context if entry.get("type") == "reasoning"]
        return len(reasoning_entries)

    def _assess_delegation_complexity(self, args: dict) -> str:
        """Assess complexity of delegation task"""
        task_desc = args.get("task_description", "")
        tools_count = len(args.get("tools_list", []))

        if tools_count > 3 or len(task_desc) > 100:
            return "high"
        elif tools_count > 1 or len(task_desc) > 50:
            return "medium"
        else:
            return "low"

    def _estimate_plan_complexity(self, goals: list) -> str:
        """Estimate complexity of plan"""
        goal_count = len(goals)
        total_text = sum(len(str(goal)) for goal in goals)

        if goal_count > 5 or total_text > 500:
            return "high"
        elif goal_count > 2 or total_text > 200:
            return "medium"
        else:
            return "low"

    def _calculate_tool_performance_score(self, duration: float, tool_name: str) -> float:
        """Calculate performance score for tool execution"""
        # Expected durations by tool type
        expected_durations = {
            "internal_reasoning": 0.1,
            "manage_internal_task_stack": 0.05,
            "delegate_to_llm_tool_node": 3.0,
            "create_and_execute_plan": 10.0,
            "direct_response": 0.1
        }

        expected = expected_durations.get(tool_name, 1.0)
        if duration <= expected:
            return 1.0
        else:
            return max(0.0, expected / duration)

    def _create_reasoning_summary(self) -> str:
        """Create summary of reasoning process"""
        reasoning_entries = [entry for entry in self.reasoning_context if entry.get("type") == "reasoning"]
        task_entries = len(self.internal_task_stack)

        return f"Completed {len(reasoning_entries)} reasoning steps with {task_entries} tasks tracked"

    def _calculate_batch_performance(self, matches: list) -> dict[str, Any]:
        """Calculate performance metrics for batch execution"""
        tool_types = [match[0] for match in matches]
        return {
            "total_tools": len(matches),
            "tool_diversity": len(set(tool_types)),
            "most_used_tool": max(set(tool_types), key=tool_types.count) if tool_types else "none"
        }

    def _assess_reasoning_progress(self) -> str:
        """Assess overall reasoning progress"""
        if len(self.reasoning_context) < 3:
            return "early_stage"
        elif len(self.reasoning_context) < 8:
            return "developing"
        elif len(self.reasoning_context) < 15:
            return "mature"
        else:
            return "extensive"

    def _get_error_context(self, error: Exception) -> dict[str, Any]:
        """Get contextual information about an error"""
        return {
            "error_class": type(error).__name__,
            "reasoning_stage": f"loop_{self.current_loop_count}",
            "context_available": len(self.reasoning_context) > 0,
            "stack_state": "populated" if self.internal_task_stack else "empty"
        }

    async def _execute_internal_reasoning(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute internal reasoning meta-tool"""
        thought = args.get("thought", "")
        thought_number = args.get("thought_number", 1)
        total_thoughts = args.get("total_thoughts", 1)
        next_thought_needed = args.get("next_thought_needed", False)
        current_focus = args.get("current_focus", "")
        key_insights = args.get("key_insights", [])
        potential_issues = args.get("potential_issues", [])
        confidence_level = args.get("confidence_level", 0.5)

        # Structure the reasoning entry
        reasoning_entry = {
            "thought": thought,
            "thought_number": thought_number,
            "total_thoughts": total_thoughts,
            "next_thought_needed": next_thought_needed,
            "current_focus": current_focus,
            "key_insights": key_insights,
            "potential_issues": potential_issues,
            "confidence_level": confidence_level,
            "timestamp": datetime.now().isoformat()
        }

        # Add to internal reasoning log
        if not hasattr(self, 'internal_reasoning_log'):
            self.internal_reasoning_log = []
        self.internal_reasoning_log.append(reasoning_entry)

        # Format for context
        context_addition = f"""Internal Reasoning Step {thought_number}/{total_thoughts}:
Thought: {thought}
Focus: {current_focus}
Key Insights: {', '.join(key_insights) if key_insights else 'None'}
Potential Issues: {', '.join(potential_issues) if potential_issues else 'None'}
Confidence: {confidence_level}
Next Thought Needed: {next_thought_needed}"""

        return {"context_addition": context_addition}

    async def _execute_manage_task_stack(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute task stack management meta-tool"""
        action = args.get("action", "get_current").lower()
        task_description = args.get("task_description", "")

        if action == "add":
            self.internal_task_stack.append({
                "description": task_description,
                "status": "pending",
                "added_at": datetime.now().isoformat()
            })
            context_addition = f"Added to task stack: {task_description}"

        elif action == "remove":
            # Remove task by description match
            original_count = len(self.internal_task_stack)
            self.internal_task_stack = [
                task for task in self.internal_task_stack
                if task_description.lower() not in task["description"].lower()
            ]
            removed_count = original_count - len(self.internal_task_stack)
            context_addition = f"Removed {removed_count} task(s) matching: {task_description}"

        elif action == "complete":
            # Mark task as completed
            for task in self.internal_task_stack:
                if task_description.lower() in task["description"].lower():
                    task["status"] = "completed"
                    task["completed_at"] = datetime.now().isoformat()
            context_addition = f"Marked as completed: {task_description}"

        elif action == "get_current":
            if self.internal_task_stack:
                stack_summary = []
                for i, task in enumerate(self.internal_task_stack):
                    status = task["status"]
                    desc = task["description"]
                    stack_summary.append(f"{i + 1}. [{status.upper()}] {desc}")
                context_addition = "Current task stack:\n" + "\n".join(stack_summary)
            else:
                context_addition = "Task stack is empty"

        else:
            context_addition = f"Unknown task stack action: {action}"

        return {"context_addition": context_addition}

    async def _execute_delegate_llm_tool(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute delegation to LLMToolNode"""
        task_description = args.get("task_description", "")
        tools_list = args.get("tools_list", [])

        # Prepare shared state for LLMToolNode
        llm_tool_shared = {
            "current_task_description": task_description + '\nreturn all results in the final answer!',
            "current_query": task_description,
            "formatted_context": {
                "recent_interaction": f"Reasoner delegating task: {task_description}",
                "session_summary": self._get_reasoning_summary(),
                "task_context": f"Reasoning loop {self.current_loop_count}, delegated task. return all results!"
            },
            "variable_manager": prep_res.get("variable_manager"),
            "agent_instance": prep_res.get("agent_instance"),
            "available_tools": tools_list,  # Restrict to specific tools
            "tool_capabilities": prep_res.get("tool_capabilities", {}),
            "fast_llm_model": prep_res.get("fast_llm_model"),
            "complex_llm_model": prep_res.get("complex_llm_model"),
            "progress_tracker": prep_res.get("progress_tracker"),
            "session_id": prep_res.get("session_id"),
            "use_fast_response": True  # Use fast model for delegated tasks
        }

        # Execute LLMToolNode
        try:
            llm_tool_node = LLMToolNode()
            await llm_tool_node.run_async(llm_tool_shared)

            # Get results
            final_response = llm_tool_shared.get("current_response", "Task completed without specific result")
            tool_calls_made = llm_tool_shared.get("tool_calls_made", 0)

            context_addition = f"""Delegated Task Completed:
Task: {task_description}
Tools Available: {', '.join(tools_list)}
Tools Used: {tool_calls_made} tool calls made
Result: {final_response}"""

            return {"context_addition": context_addition}

        except Exception as e:
            context_addition = f"Delegation failed for task '{task_description}': {str(e)}"
            return {"context_addition": context_addition}

    async def _execute_create_plan(self, args: dict, prep_res: dict) -> dict[str, Any]:
        """Execute plan creation and execution"""
        goals = args.get("goals", [])

        if not goals:
            return {"context_addition": "No goals provided for plan creation"}

        try:
            # Prepare shared state for TaskPlanner
            planning_shared = prep_res.copy()
            planning_shared.update({
                "replan_context": {
                    "goals": goals,
                    "triggered_by": "llm_reasoner",
                    "reasoning_context": self._get_reasoning_summary()
                },
                "current_task_description": f"Execute plan with {len(goals)} goals",
                "current_query": f"Complex task: {'; '.join(goals)}"
            })

            # Execute TaskPlanner
            planner_node = TaskPlannerNode()
            plan_info = await planner_node.run_async(planning_shared)

            if plan_info == "planning_failed":
                return {"context_addition": f"Plan creation failed: {planning_shared.get('planning_error', 'Unknown error')}"}

            plan = planning_shared.get("current_plan")
            # Execute the plan using TaskExecutor
            executor_shared = planning_shared.copy()
            executor_node = TaskExecutorNode()

            # Execute plan to completion
            max_execution_cycles = 10
            execution_cycle = 0

            while execution_cycle < max_execution_cycles:
                execution_cycle += 1

                result = await executor_node.run_async(executor_shared)

                # Check completion status
                if result == "plan_completed" or result == "execution_error":
                    break
                elif result in ["continue_execution", "waiting"]:
                    continue
                else:
                    # Handle other results like reflection needs
                    if result in ["needs_dynamic_replan", "needs_plan_append"]:
                        # For now, just continue - could add reflection logic here
                        continue
                    break

            # Collect results
            completed_tasks = [
                task for task in plan.tasks
                if executor_shared["tasks"][task.id].status == "completed"
            ]

            failed_tasks = [
                task for task in plan.tasks
                if executor_shared["tasks"][task.id].status == "failed"
            ]

            # Build context addition with results
            results_summary = []
            results_store = executor_shared.get("results", {})

            for task in completed_tasks:
                task_result = results_store.get(task.id, {})
                if task_result.get("data"):
                    result_preview = str(task_result["data"])[:150] + "..."
                    results_summary.append(f"- {task.description}: {result_preview}")

            context_addition = f"""Plan Execution Completed:
Goals: {len(goals)} goals processed
Tasks Created: {len(plan.tasks)}
Tasks Completed: {len(completed_tasks)}
Tasks Failed: {len(failed_tasks)}
Execution Cycles: {execution_cycle}

Results Summary:
{chr(10).join(results_summary) if results_summary else 'No specific results captured'}"""

            return {"context_addition": context_addition}

        except Exception as e:
            import traceback
            print(traceback.format_exc())
            context_addition = f"Plan execution failed: {str(e)}"
            return {"context_addition": context_addition}

    def _get_reasoning_summary(self) -> str:
        """Get a summary of the reasoning process so far"""
        if not self.reasoning_context:
            return "No reasoning context available"

        summary_parts = []
        reasoning_entries = [entry for entry in self.reasoning_context if entry.get("type") == "reasoning"]

        for entry in reasoning_entries[-3:]:  # Last 3 reasoning steps
            content = entry.get("content", "")[:50000] + "..."
            loop_num = entry.get("loop", "?")
            summary_parts.append(f"Loop {loop_num}: {content}")

        return "\n".join(summary_parts)

    async def _create_error_response(self, query: str, error: str) -> str:
        """Create an error response"""
        return f"I encountered an error while processing your request: {error}. I was working on: {query}"

    async def _fallback_direct_response(self, prep_res: dict) -> dict[str, Any]:
        """Fallback when LLM is not available"""
        query = prep_res["original_query"]
        fallback_response = f"I received your request: {query}. However, I'm currently unable to process complex requests due to limited capabilities."

        return {
            "final_result": fallback_response,
            "reasoning_loops": 0,
            "reasoning_context": [{"type": "fallback", "content": "LLM unavailable"}],
            "internal_task_stack": []
        }
exec_async(prep_res) async

Enhanced main reasoning loop with outline-driven execution

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
async def exec_async(self, prep_res):
    """Enhanced main reasoning loop with outline-driven execution"""
    if not LITELLM_AVAILABLE:
        return await self._fallback_direct_response(prep_res)

    original_query = prep_res["original_query"]
    agent_instance = prep_res["agent_instance"]
    progress_tracker = prep_res.get("progress_tracker")

    # Initialize enhanced reasoning context
    await self._initialize_reasoning_session(prep_res, original_query)

    # STEP 1: MANDATORY OUTLINE CREATION
    if not self.outline:
        with Spinner("Creating initial outline..."):
            outline_result = await self._create_initial_outline(prep_res)
        if self.outline and len(self.outline.get("steps", [])) == 1:
            # fast llm respose on the input metoning tis is a direct respose and evalute if the input dosent need an outline
            print("Fast direct response triggered")
            response = await self.agent_instance.a_run_llm_completion(
                model=prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku"),
                messages=[{"role": "user", "content": prep_res["original_query"]}],
                temperature=0.3,
                max_tokens=2048,
                node_name="LLMReasonerNode",
                task_id="fast_direct_response"
            )
            return {
                    "final_result": response,
                    "reasoning_loops": self.current_loop_count,
                    "reasoning_context": self.reasoning_context.copy(),
                    "internal_task_stack": self.internal_task_stack.copy(),
                    "outline": self.outline,
                    "outline_completion": self.current_outline_step,
                    "performance_metrics": self.performance_metrics,
                    "auto_recovery_attempts": self.auto_recovery_attempts
                }
        elif not outline_result:
            return await self._fallback_direct_response(prep_res)

    final_result = None
    consecutive_no_progress = 0
    max_no_progress = 3

    # Enhanced main reasoning loop with strict progress tracking
    while self.current_reasoning_count < self.max_reasoning_loops:
        self.current_loop_count += 1
        loop_start_time = time.time()

        # Check for infinite loops
        if self._detect_infinite_loop():
            await self._trigger_auto_recovery(prep_res)
            if self.auto_recovery_attempts >= self.max_auto_recovery:
                break

        # Auto-context management
        await self._manage_context_size()

        # Progress tracking
        if progress_tracker:
            await progress_tracker.emit_event(ProgressEvent(
                event_type="reasoning_loop",
                timestamp=time.time(),
                node_name="LLMReasonerNode",
                status=NodeStatus.RUNNING,
                metadata={
                    "loop_number": self.current_loop_count,
                    "outline_step": self.current_outline_step,
                    "outline_total": len(self.outline.get("steps", [])) if self.outline else 0,
                    "context_size": len(self.reasoning_context),
                    "task_stack_size": len(self.internal_task_stack),
                    "auto_recovery_attempts": self.auto_recovery_attempts,
                    "performance_metrics": self.performance_metrics
                }
            ))

        try:
            # Build enhanced reasoning prompt with outline context
            reasoning_prompt = await self._build_outline_driven_prompt(prep_res)

            # Force progress check if needed
            if self.mandatory_progress_check and consecutive_no_progress >= 2:
                reasoning_prompt += "\n\n**MANDATORY**: You must either complete current outline step or move to next step. No more analysis without action!"

            # LLM reasoning call
            model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")

            llm_response = await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": reasoning_prompt}],
                temperature=0.2,  # Lower temperature for more focused execution
                # max_tokens=3072,
                node_name="LLMReasonerNode",
                stop="<immediate_context>",
                task_id=f"reasoning_loop_{self.current_loop_count}_step_{self.current_outline_step}"
            )

            # Add LLM response to context
            self.reasoning_context.append({
                "type": "reasoning",
                "content": llm_response,
                "loop": self.current_loop_count,
                "outline_step": self.current_outline_step,
                "timestamp": datetime.now().isoformat()
            })

            # Parse and execute meta-tool calls with enhanced tracking
            progress_made = await self._parse_and_execute_meta_tools(llm_response, prep_res)

            action_taken = progress_made.get("action_taken", False)
            actual_progress = progress_made.get("progress_made", False)

            # Update performance with correct progress indication
            self._update_performance_metrics(loop_start_time, actual_progress)

            if not action_taken:
                self.current_reasoning_count += 1
                if self.current_outline_step > len(self.outline.get("steps", [])):
                    progress_made["final_result"] = llm_response
                    rprint("Final result reached forced by outline step count")
                if self.current_outline_step < len(self.outline.get("steps", [])) and self.outline.get("steps", [])[self.current_outline_step].get("is_final", False):
                    progress_made["final_result"] = llm_response
                    rprint("Final result reached forced by outline step count final step")
            else:
                self.current_reasoning_count -= 1

            # Check for final result
            if progress_made.get("final_result"):
                final_result = progress_made["final_result"]
                await self._finalize_reasoning_session(prep_res, final_result)
                break

            # Progress monitoring
            if progress_made.get("action_taken"):
                consecutive_no_progress = 0
                self._update_performance_metrics(loop_start_time, True)
            else:
                consecutive_no_progress += 1
                self._update_performance_metrics(loop_start_time, False)

            # Check outline completion
            if self.outline and self.current_outline_step >= len(self.outline.get("steps", []))+self.max_reasoning_loops:
                # All outline steps completed, force final response
                final_result = await self._create_outline_completion_response(prep_res)
                break

            # Emergency break for excessive no-progress
            if consecutive_no_progress >= max_no_progress:
                await self._trigger_auto_recovery(prep_res)

        except Exception as e:
            await self._handle_reasoning_error(e, prep_res, progress_tracker)
            import traceback
            print(traceback.format_exc())
            if self.auto_recovery_attempts >= self.max_auto_recovery:
                final_result = await self._create_error_response(original_query, str(e))
                break


    # If no final result after max loops, create a comprehensive summary
    if not final_result:
        final_result = await self._create_enhanced_timeout_response(original_query, prep_res)

    return {
        "final_result": final_result,
        "reasoning_loops": self.current_loop_count,
        "reasoning_context": self.reasoning_context.copy(),
        "internal_task_stack": self.internal_task_stack.copy(),
        "outline": self.outline,
        "outline_completion": self.current_outline_step,
        "performance_metrics": self.performance_metrics,
        "auto_recovery_attempts": self.auto_recovery_attempts
    }
post_async(shared, prep_res, exec_res) async

Enhanced post-processing with comprehensive data storage

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
6687
6688
6689
6690
6691
6692
6693
6694
6695
6696
6697
6698
6699
6700
6701
6702
6703
6704
6705
6706
6707
6708
6709
6710
6711
6712
6713
6714
6715
6716
6717
6718
6719
6720
6721
6722
6723
6724
6725
6726
6727
6728
6729
6730
6731
6732
6733
6734
async def post_async(self, shared, prep_res, exec_res):
    """Enhanced post-processing with comprehensive data storage"""
    final_result = exec_res.get("final_result", "Task processing incomplete")

    # Store comprehensive reasoning artifacts
    shared["reasoning_artifacts"] = {
        "reasoning_loops": exec_res.get("reasoning_loops", 0),
        "reasoning_context": exec_res.get("reasoning_context", []),
        "internal_task_stack": exec_res.get("internal_task_stack", []),
        "outline": exec_res.get("outline"),
        "outline_completion": exec_res.get("outline_completion", 0),
        "performance_metrics": exec_res.get("performance_metrics", {}),
        "auto_recovery_attempts": exec_res.get("auto_recovery_attempts", 0)
    }

    # Enhanced variable system updates
    if self.variable_manager:
        # Store final session results
        final_session_data = {
            "final_result": final_result,
            "completion_timestamp": datetime.now().isoformat(),
            "total_loops": exec_res.get("reasoning_loops", 0),
            "session_success": final_result != "Task processing incomplete",
            "outline_driven_execution": True
        }
        self.variable_manager.set("reasoning.current_session.final_data", final_session_data)

        # Update global performance statistics
        self._update_global_performance_stats(exec_res)

    # Set enhanced response data
    shared["llm_reasoner_result"] = final_result
    shared["current_response"] = final_result

    # Provide enhanced synthesis metadata
    shared["synthesized_response"] = {
        "synthesized_response": final_result,
        "confidence": self._calculate_confidence(exec_res),
        "metadata": {
            "synthesis_method": "outline_driven_reasoner",
            "reasoning_loops": exec_res.get("reasoning_loops", 0),
            "outline_completion": exec_res.get("outline_completion", 0),
            "performance_score": self._calculate_performance_score(exec_res),
            "auto_recovery_used": exec_res.get("auto_recovery_attempts", 0) > 0
        }
    }

    return "reasoner_complete"
prep_async(shared) async

Enhanced initialization with variable system integration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
async def prep_async(self, shared):
    """Enhanced initialization with variable system integration"""
    # Reset for new execution
    self.reasoning_context = []
    self.internal_task_stack = []
    self.current_loop_count = 0
    self.current_reasoning_count = 0
    self.outline = None
    self.current_outline_step = 0
    self.step_completion_tracking = {}
    self.loop_detection_memory = []
    self.performance_metrics = {
        "loop_times": [],
        "progress_loops": 0,
        "total_loops": 0
    }
    self.auto_recovery_attempts = 0
    self.last_action_signatures = []

    self.agent_instance = shared.get("agent_instance")

    # Enhanced variable manager integration
    self.variable_manager = shared.get("variable_manager", self.agent_instance.variable_manager)
    context_manager = shared.get("context_manager")

    if self.variable_manager:
        # Store reasoning session context
        session_context = {
            "session_id": shared.get("session_id", "default"),
            "start_time": datetime.now().isoformat(),
            "query": shared.get("current_query", ""),
            "reasoning_mode": "outline_driven"
        }
        self.variable_manager.set("reasoning.current_session", session_context)
        # Load previous successful patterns from variables
        self._load_historical_patterns()

    #Build comprehensive system context via UnifiedContextManager
    system_context = await self._build_enhanced_system_context_unified(shared, context_manager)

    return {
        "original_query": shared.get("current_query", ""),
        "session_id": shared.get("session_id", "default"),
        "agent_instance": shared.get("agent_instance"),
        "variable_manager": self.variable_manager,
        "context_manager": context_manager,  #Context Manager Reference
        "system_context": system_context,
        "available_tools": shared.get("available_tools", []),
        "tool_capabilities": shared.get("tool_capabilities", {}),
        "fast_llm_model": shared.get("fast_llm_model"),
        "complex_llm_model": shared.get("complex_llm_model"),
        "progress_tracker": shared.get("progress_tracker"),
        "formatted_context": shared.get("formatted_context", {}),
        "historical_context": await self._get_historical_context_unified(context_manager, shared.get("session_id")),
        "capabilities_summary": shared.get("capabilities_summary", ""),
        # Sub-system references
        "llm_tool_node": shared.get("llm_tool_node_instance"),
        "task_planner": shared.get("task_planner_instance"),
        "task_executor": shared.get("task_executor_instance"),
    }
LLMTask dataclass

Bases: Task

Spezialisierter Task für LLM-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py
475
476
477
478
479
480
481
482
483
484
485
@dataclass
class LLMTask(Task):
    """Spezialisierter Task für LLM-Aufrufe"""
    llm_config: dict[str, Any] = field(default_factory=lambda: {
        "model_preference": "fast",  # "fast" | "complex"
        "temperature": 0.7,
        "max_tokens": 1024
    })
    prompt_template: str = ""
    context_keys: list[str] = field(default_factory=list)  # Keys aus shared state
    output_schema: dict  = None  # JSON Schema für Validierung
LLMToolNode

Bases: AsyncNode

Enhanced LLM tool with automatic tool calling and agent loop integration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
@with_progress_tracking
class LLMToolNode(AsyncNode):
    """Enhanced LLM tool with automatic tool calling and agent loop integration"""

    def __init__(self, model: str = None, max_tool_calls: int = 5, **kwargs):
        super().__init__(**kwargs)
        self.model = model or os.getenv("COMPLEXMODEL", "openrouter/qwen/qwen3-code")
        self.max_tool_calls = max_tool_calls
        self.call_log = []

    async def prep_async(self, shared):
        context = shared.get("formatted_context", {})
        task_description = shared.get("current_task_description", shared.get("current_query", ""))

        # Variable Manager integration
        variable_manager = shared.get("variable_manager")
        agent_instance = shared.get("agent_instance")

        return {
            "task_description": task_description,
            "context": context,
            "context_manager": shared.get("context_manager"),
            "session_id": shared.get("session_id"),
            "variable_manager": variable_manager,
            "agent_instance": agent_instance,
            "available_tools": shared.get("available_tools", []),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "persona_config": shared.get("persona_config"),
            "base_system_message": variable_manager.format_text(agent_instance.amd.get_system_message_with_persona()),
            "recent_interaction": context.get("recent_interaction", ""),
            "session_summary": context.get("session_summary", ""),
            "task_context": context.get("task_context", ""),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "progress_tracker": shared.get("progress_tracker"),
            "tool_call_count": 0
        }

    async def exec_async(self, prep_res):
        """Main execution with tool calling loop"""
        if not LITELLM_AVAILABLE:
            return await self._fallback_response(prep_res)

        progress_tracker = prep_res.get("progress_tracker")

        conversation_history = []
        tool_call_count = 0
        final_response = None
        model_to_use = "auto"
        total_llm_calls = 0
        total_cost = 0.0
        total_tokens = 0

        # Initial system message with tool awareness
        system_message = self._build_tool_aware_system_message(prep_res)

        # Initial user prompt with variable resolution
        initial_prompt = await self._build_context_aware_prompt(prep_res)
        conversation_history.append({"role": "user", "content":  prep_res["variable_manager"].format_text(initial_prompt)})
        runs = 0
        while tool_call_count < self.max_tool_calls:
            runs += 1
            # Get LLM response
            messages = [{"role": "system", "content": system_message + ( "\nfist look at the context and reason over you intal step." if runs == 1 else "")}] + conversation_history

            model_to_use = self._select_optimal_model(prep_res["task_description"], prep_res)

            llm_start = time.perf_counter()

            try:
                agent_instance = prep_res["agent_instance"]
                response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=messages,
                    temperature=0.7,
                    stream=False,
                    # max_tokens=2048,
                    node_name="LLMToolNode", task_id="llm_phase_" + str(runs)
                )

                llm_response = response
                if not llm_response and  not final_response:
                    final_response = "I encountered an error while processing your request."
                    break


                # Check for tool calls
                tool_calls = self._extract_tool_calls(llm_response)

                llm_response = prep_res["variable_manager"].format_text(llm_response)
                conversation_history.append({"role": "assistant", "content": llm_response})


                if not tool_calls:
                    # No more tool calls, this is the final response
                    final_response = llm_response
                    break

                # Execute tool calls
                tool_results = await self._execute_tool_calls(tool_calls, prep_res)
                tool_call_count += len(tool_calls)

                # Add tool results to conversation
                tool_results_text = self._format_tool_results(tool_results)
                final_response = tool_results_text
                conversation_history.append({"role": "user",
                                             "content": f"Tool results:\n{tool_results_text}\n\nPlease continue with the next action do nor repeat or provide your final response."})

                # Update variable manager with tool results
                self._update_variables_with_results(tool_results, prep_res["variable_manager"])

            except Exception as e:
                llm_duration = time.perf_counter() - llm_start

                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="llm_call",  # Konsistenter Event-Typ
                        node_name="LLMToolNode",
                        session_id=prep_res.get("session_id"),
                        status=NodeStatus.FAILED,
                        success=False,
                        duration=llm_duration,
                        llm_model=model_to_use,
                        error_details={
                            "message": str(e),
                            "type": type(e).__name__
                        },
                        metadata={"call_number": total_llm_calls + 1}
                    ))
                eprint(f"LLM tool execution failed: {e}")
                final_response = f"I encountered an error while processing: {str(e)}"
                import traceback
                print(traceback.format_exc())
                break


        return {
            "success": True,
            "final_response": final_response or "I was unable to complete the request.",
            "tool_calls_made": tool_call_count,
            "conversation_history": conversation_history,
            "model_used": model_to_use,
            "llm_statistics": {
                "total_calls": total_llm_calls,
                "total_cost": total_cost,
                "total_tokens": total_tokens
            }
        }

    def _build_tool_aware_system_message(self, prep_res: dict) -> str:
        """Build a unified intelligent, tool-aware system message with context and relevance analysis."""

        # Base system message
        base_message = prep_res.get("base_system_message", "You are a helpful AI assistant.")
        available_tools = prep_res.get("available_tools", [])
        tool_capabilities = prep_res.get("tool_capabilities", {})
        variable_manager = prep_res.get("variable_manager")
        context = prep_res.get("context", {})
        agent_instance = prep_res.get("agent_instance")
        query = prep_res.get('task_description', '').lower()

        base_message += ("\n\nAlways follow this action pattern"
                         "**THINK** -> **PLAN** -> **ACT** using tools!\n"
                         "all progress must be stored to ( variable system, memory, external services )!\n"
                         "if working on code or file based tasks, update and crate the files! sve result in file!\n")

        # --- Part 1: List available tools & capabilities ---
        if available_tools:
            base_message += f"\n\n## Available Tools\nYou have access to these tools: {', '.join(available_tools)}\n"
            base_message += "Results will be stored to results.{tool_name}.data"

            for tool_name in available_tools:
                if tool_name in tool_capabilities:
                    cap = tool_capabilities[tool_name]
                    base_message += f"\n**{tool_name}**: {cap.get('primary_function', 'No description')}"
                    use_cases = cap.get('use_cases', [])
                    if use_cases:
                        base_message += f"\n  Use cases: {', '.join(use_cases[:3])}"

            # base_message += "\n\n## Tool Usage\nTo use tools, respond with:\nTOOL_CALL: tool_name(arg1='value1', arg2='value2')\nYou can make multiple tool calls in one response."
            base_message += """
## Tool Usage
To use tools, respond with a YAML block:
```yaml
TOOL_CALLS:
  - tool: tool_name
    args:
      arg1: value1
      arg2: value2
  - tool: another_tool
    args:
      code: |
        def example():
            return "multi-line code"
      text: |
        Multi-line text
        with arbitrary content
```
You can call multiple tools in one response. Use | for multi-line strings containing code or complex text."""

        # --- Part 2: Add variable context ---
        if variable_manager:
            var_context = variable_manager.get_llm_variable_context()
            if var_context:
                base_message += f"\n\n## Variable Context\n{var_context}"

        # --- Part 3: Intelligent tool analysis ---
        if not agent_instance or not hasattr(agent_instance, '_tool_capabilities'):
            return base_message + "\n\n⚠ No intelligent tool analysis available."

        capabilities = agent_instance._tool_capabilities
        analysis_parts = ["\n\n## Intelligent Tool Analysis"]

        for tool_name, cap in capabilities.items():
            analysis_parts.append(f"\n{tool_name}{cap.get('args_schema', '()')}:")
            analysis_parts.append(f"- Function: {cap.get('primary_function', 'Unknown')}")

            # Calculate relevance score
            relevance_score = self._calculate_tool_relevance(query, cap)
            analysis_parts.append(f"- Query relevance: {relevance_score:.2f}")

            if relevance_score > 0.65:
                analysis_parts.append("- ⭐ HIGHLY RELEVANT - SHOULD USE THIS TOOL!")

            # Trigger phrase matching
            triggers = cap.get('trigger_phrases', [])
            matched_triggers = [t for t in triggers if t.lower() in query]
            if matched_triggers:
                analysis_parts.append(f"- Matched triggers: {matched_triggers}")

            # Show top use cases
            use_cases = cap.get('use_cases', [])[:3]
            if use_cases:
                analysis_parts.append(f"- Use cases: {', '.join(use_cases)}")

        # Combine everything into a final message
        return base_message + "\n"+ "\n".join(analysis_parts)

    def _calculate_tool_relevance(self, query: str, capabilities: dict) -> float:
        """Calculate how relevant a tool is to the current query"""

        query_words = set(query.lower().split())

        # Check trigger phrases
        trigger_score = 0.0
        triggers = capabilities.get('trigger_phrases', [])
        for trigger in triggers:
            trigger_words = set(trigger.lower().split())
            if trigger_words.intersection(query_words):
                trigger_score += 0.04
        # Check confidence triggers if available
        conf_triggers = capabilities.get('confidence_triggers', {})
        for phrase, confidence in conf_triggers.items():
            if phrase.lower() in query:
                trigger_score += confidence/10
        # Check indirect connections
        indirect = capabilities.get('indirect_connections', [])
        for connection in indirect:
            connection_words = set(connection.lower().split())
            if connection_words.intersection(query_words):
                trigger_score += 0.02
        return min(1.0, trigger_score)

    @staticmethod
    def _extract_tool_calls_custom(text: str) -> list[dict]:
        """Extract tool calls from LLM response"""

        tool_calls = []

        pattern = r'TOOL_CALL:'
        matches = _extract_meta_tool_calls(text, pattern)

        for tool_name, args_str in matches:
            try:
                # Parse arguments
                args = _parse_tool_args(args_str)
                tool_calls.append({
                    "tool_name": tool_name,
                    "arguments": args
                })
            except Exception as e:
                wprint(f"Failed to parse tool call {tool_name}: {e}")

        return tool_calls

    @staticmethod
    def _extract_tool_calls(text: str) -> list[dict]:
        """Extract tool calls from LLM response using YAML format"""
        import re

        import yaml

        tool_calls = []

        # Pattern to find YAML blocks with TOOL_CALLS
        yaml_pattern = r'```yaml\s*\n(.*?TOOL_CALLS:.*?)\n```'
        yaml_matches = re.findall(yaml_pattern, text, re.DOTALL | re.IGNORECASE)

        # Also try without code blocks for simpler cases
        if not yaml_matches:
            simple_pattern = r'TOOL_CALLS:\s*\n((?:.*\n)*?)(?=\n\S|\Z)'
            simple_matches = re.findall(simple_pattern, text, re.MULTILINE)
            if simple_matches:
                yaml_matches = [f"TOOL_CALLS:\n{match}" for match in simple_matches]

        for yaml_content in yaml_matches:
            try:
                # Parse YAML content
                parsed_yaml = yaml.safe_load(yaml_content)

                if not isinstance(parsed_yaml, dict) or 'TOOL_CALLS' not in parsed_yaml:
                    continue

                calls = parsed_yaml['TOOL_CALLS']
                if not isinstance(calls, list):
                    calls = [calls]  # Handle single tool call

                for call in calls:
                    if isinstance(call, dict) and 'tool' in call:
                        tool_call = {
                            "tool_name": call['tool'],
                            "arguments": call.get('args', {})
                        }
                        tool_calls.append(tool_call)

            except yaml.YAMLError as e:
                wprint(f"Failed to parse YAML tool calls: {e}")
            except Exception as e:
                wprint(f"Error processing tool calls: {e}")

        return tool_calls

    def _select_optimal_model(self, task_description: str, prep_res: dict) -> str:
        """Select optimal model based on task complexity and available resources"""
        complexity_score = self._estimate_task_complexity(task_description, prep_res)
        if complexity_score > 0.7:
            return prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")
        else:
            return prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")

    def _estimate_task_complexity(self, task_description: str, prep_res: dict) -> float:
        """Estimate task complexity based on description, length, and available tools"""
        # Simple heuristic: length + keyword matching + tool availability
        description_length_score = min(len(task_description) / 500, 1.0)  # cap at 1.0
        keywords = ["analyze", "research", "generate", "simulate", "complex", "deep", "strategy"]
        keyword_score = sum(1 for k in keywords if k in task_description.lower()) / len(keywords)
        tool_score = min(len(prep_res.get("available_tools", [])) / 10, 1.0)

        # Weighted sum
        complexity_score = (0.5 * description_length_score) + (0.3 * keyword_score) + (0.2 * tool_score)
        return round(complexity_score, 2)

    async def _fallback_response(self, prep_res: dict) -> dict:
        """Fallback response if LiteLLM is not available"""
        wprint("LiteLLM not available — using fallback response.")
        return {
            "success": False,
            "final_response": (
                "I'm unable to process this request fully right now because the LLM interface "
                "is not available. Please try again later or check system configuration."
            ),
            "tool_calls_made": 0,
            "conversation_history": [],
            "model_used": None
        }

    async def _execute_tool_calls(self, tool_calls: list[dict], prep_res: dict) -> list[dict]:
        """Execute tool calls via agent"""
        agent_instance = prep_res.get("agent_instance")
        variable_manager = prep_res.get("variable_manager")
        progress_tracker = prep_res.get("progress_tracker")

        results = []

        for tool_call in tool_calls:
            tool_name = tool_call["tool_name"]
            arguments = tool_call["arguments"]

            # Start tool tracking
            tool_start = time.perf_counter()

            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",
                    timestamp=time.time(),
                    status=NodeStatus.RUNNING,
                    node_name="LLMToolNode",
                    tool_name=tool_name,
                    tool_args=arguments,
                    session_id=prep_res.get("session_id"),
                    metadata={"tool_call_initiated": True}
                ))

            try:
                # Resolve variables in arguments
                if variable_manager:
                    resolved_args = {}
                    for key, value in arguments.items():
                        if isinstance(value, str):
                            resolved_args[key] = variable_manager.format_text(value)
                        else:
                            resolved_args[key] = value
                else:
                    resolved_args = arguments

                # Execute via agent
                result = await agent_instance.arun_function(tool_name, **resolved_args)
                tool_duration = time.perf_counter() - tool_start
                variable_manager.set(f"results.{tool_name}.data", result)
                results.append({
                    "tool_name": tool_name,
                    "arguments": resolved_args,
                    "success": True,
                    "result": result
                })

            except Exception as e:
                tool_duration = time.perf_counter() - tool_start
                error_message = str(e)
                error_type = type(e).__name__
                import traceback
                print(traceback.format_exc())


                if progress_tracker:
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="tool_call",
                        timestamp=time.time(),
                        node_name="LLMToolNode",
                        status=NodeStatus.FAILED,
                        tool_name=tool_name,
                        tool_args=arguments,
                        duration=tool_duration,
                        success=False,
                        tool_error=error_message,
                        session_id=prep_res.get("session_id"),
                        metadata={
                            "error": error_message,
                            "error_message": error_message,
                            "error_type": error_type
                        }
                    ))

                    # FIXED: Also send generic error event for error log
                    await progress_tracker.emit_event(ProgressEvent(
                        event_type="error",
                        timestamp=time.time(),
                        node_name="LLMToolNode",
                        status=NodeStatus.FAILED,
                        success=False,
                        tool_name=tool_name,
                        metadata={
                            "error": error_message,
                            "error_message": error_message,
                            "error_type": error_type,
                            "source": "tool_execution",
                            "tool_name": tool_name,
                            "tool_args": arguments
                        }
                    ))
                eprint(f"Tool execution failed {tool_name}: {e}")
                results.append({
                    "tool_name": tool_name,
                    "arguments": arguments,
                    "success": False,
                    "error": str(e)
                })

        return results

    def _format_tool_results(self, results: list[dict]) -> str:
        """Format tool results for LLM"""
        formatted = []

        for result in results:
            if result["success"]:
                formatted.append(f"✓ {result['tool_name']}: {result['result']}")
            else:
                formatted.append(f"✗ {result['tool_name']}: ERROR - {result['error']}")

        return "\n".join(formatted)

    def _update_variables_with_results(self, results: list[dict], variable_manager):
        """Update variable manager with tool results"""
        if not variable_manager:
            return

        for i, result in enumerate(results):
            if result["success"]:
                tool_name = result['tool_name']
                result_data = result['result']

                # FIXED: Store result in proper variable paths
                variable_manager.set(f"results.{tool_name}.data", result_data)
                variable_manager.set(f"tools.{tool_name}.result", result_data)

                # Also store with index for multiple calls to same tool
                var_key = f"tool_result_{tool_name}_{i}"
                variable_manager.set(var_key, result_data)

    async def _build_context_aware_prompt(self, prep_res: dict) -> str:
        """Build context-aware prompt mit UnifiedContextManager Integration"""
        variable_manager = prep_res.get("variable_manager")
        agent_instance = prep_res.get("agent_instance")
        context = prep_res.get("context", {})

        #Get unified context if available
        context_manager = prep_res.get("context_manager")
        session_id = prep_res.get("session_id", "default")

        unified_context_parts = []

        if context_manager:
            try:
                # Get unified context für LLM Tool usage
                unified_context = await context_manager.build_unified_context(session_id, prep_res.get("task_description", ""))

                # Format unified context for LLM consumption
                chat_history = unified_context.get("chat_history", [])
                if chat_history:
                    unified_context_parts.append("## Recent Conversation from Session")
                    for msg in chat_history[-5:]:  # Last 5 messages
                        timestamp = msg.get('timestamp', '')[:19]
                        role = msg.get('role', 'unknown')
                        content = msg.get('content', '')[:300] + ("..." if len(msg.get('content', '')) > 300 else "")
                        unified_context_parts.append(f"[{timestamp}] {role}: {content}")

                # Execution state from unified context
                execution_state = unified_context.get("execution_state", {})
                if execution_state:
                    system_status = execution_state.get('system_status', 'unknown')
                    active_tasks = execution_state.get('active_tasks', [])
                    recent_completions = execution_state.get('recent_completions', [])

                    unified_context_parts.append("\n## Current System State")
                    unified_context_parts.append(f"Status: {system_status}")
                    if active_tasks:
                        unified_context_parts.append(f"Active Tasks: {len(active_tasks)}")
                    if recent_completions:
                        unified_context_parts.append(f"Recent Completions: {len(recent_completions)}")

                # Available results from unified context
                variables_context = unified_context.get("variables", {})
                recent_results = variables_context.get("recent_results", [])
                if recent_results:
                    unified_context_parts.append("\n## Available Results")
                    for result in recent_results[:3]:  # Top 3 results
                        task_id = result.get("task_id", "unknown")
                        preview = result.get("preview", "")[:100] + "..."
                        success = "✅" if result.get("success") else "❌"
                        unified_context_parts.append(f"{success=} {task_id}: {preview=}")

                # World model facts from unified context
                relevant_facts = unified_context.get("relevant_facts", [])
                if relevant_facts:
                    unified_context_parts.append("\n## Relevant Known Facts")
                    for key, value in relevant_facts:  # Top 3 facts
                        fact_preview = str(value)
                        unified_context_parts.append(f"- {key}: {fact_preview}")

            except Exception as e:
                unified_context_parts.append(f"## Context Error\nUnified context unavailable: {str(e)}")

        # EXISTIEREND: Keep existing context building (backwards compatibility)
        prompt_parts = []

        # Add unified context first (primary)
        if unified_context_parts:
            prompt_parts.extend(unified_context_parts)

        # Add existing context sections (secondary)
        recent_interaction = prep_res.get("recent_interaction", "")
        session_summary = prep_res.get("session_summary", "")
        task_context = prep_res.get("task_context", "")

        if recent_interaction:
            prompt_parts.append(f"\n## Recent Interaction Context\n{recent_interaction}")
            prompt_parts.append("\n**Important**: NO META_TOOL_CALLs needed in this section! and not avalabel\n use tools from Intelligent Tool Analysis only!")
        if session_summary:
            prompt_parts.append(f"\n## Session Summary\n{session_summary}")
        if task_context:
            prompt_parts.append(f"\n## Task Context\n{task_context}")

        # Add main task
        task_description = prep_res.get("task_description", "")
        if task_description:
            prompt_parts.append(f"\n## Current Request\n{task_description}")

        # Variable suggestions (existing functionality)
        if variable_manager and task_description:
            suggestions = variable_manager.get_variable_suggestions(task_description)
            if suggestions:
                prompt_parts.append(f"\n## Available Variables\nYou can use: {', '.join(suggestions)}")

        # Final variable resolution
        final_prompt = "\n".join(prompt_parts)
        if variable_manager:
            final_prompt = variable_manager.format_text(final_prompt)

        return final_prompt

    async def post_async(self, shared, prep_res, exec_res):
        shared["current_response"] = exec_res.get("final_response", "Task completed.")
        shared["tool_calls_made"] = exec_res.get("tool_calls_made", 0)
        shared["llm_tool_conversation"] = exec_res.get("conversation_history", [])
        shared["synthesized_response"] = {"synthesized_response":exec_res.get("final_response", "Task completed."),
                                          "confidence": (0.7 if exec_res.get("model_used") == prep_res.get("complex_llm_model") else 0.6) if exec_res.get("success", False) else 0,
                                          "metadata": exec_res.get("metadata", {"model_used": exec_res.get("model_used")}),
                                          "synthesis_method": "llm_tool"}
        return "llm_tool_complete"
exec_async(prep_res) async

Main execution with tool calling loop

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
async def exec_async(self, prep_res):
    """Main execution with tool calling loop"""
    if not LITELLM_AVAILABLE:
        return await self._fallback_response(prep_res)

    progress_tracker = prep_res.get("progress_tracker")

    conversation_history = []
    tool_call_count = 0
    final_response = None
    model_to_use = "auto"
    total_llm_calls = 0
    total_cost = 0.0
    total_tokens = 0

    # Initial system message with tool awareness
    system_message = self._build_tool_aware_system_message(prep_res)

    # Initial user prompt with variable resolution
    initial_prompt = await self._build_context_aware_prompt(prep_res)
    conversation_history.append({"role": "user", "content":  prep_res["variable_manager"].format_text(initial_prompt)})
    runs = 0
    while tool_call_count < self.max_tool_calls:
        runs += 1
        # Get LLM response
        messages = [{"role": "system", "content": system_message + ( "\nfist look at the context and reason over you intal step." if runs == 1 else "")}] + conversation_history

        model_to_use = self._select_optimal_model(prep_res["task_description"], prep_res)

        llm_start = time.perf_counter()

        try:
            agent_instance = prep_res["agent_instance"]
            response = await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=messages,
                temperature=0.7,
                stream=False,
                # max_tokens=2048,
                node_name="LLMToolNode", task_id="llm_phase_" + str(runs)
            )

            llm_response = response
            if not llm_response and  not final_response:
                final_response = "I encountered an error while processing your request."
                break


            # Check for tool calls
            tool_calls = self._extract_tool_calls(llm_response)

            llm_response = prep_res["variable_manager"].format_text(llm_response)
            conversation_history.append({"role": "assistant", "content": llm_response})


            if not tool_calls:
                # No more tool calls, this is the final response
                final_response = llm_response
                break

            # Execute tool calls
            tool_results = await self._execute_tool_calls(tool_calls, prep_res)
            tool_call_count += len(tool_calls)

            # Add tool results to conversation
            tool_results_text = self._format_tool_results(tool_results)
            final_response = tool_results_text
            conversation_history.append({"role": "user",
                                         "content": f"Tool results:\n{tool_results_text}\n\nPlease continue with the next action do nor repeat or provide your final response."})

            # Update variable manager with tool results
            self._update_variables_with_results(tool_results, prep_res["variable_manager"])

        except Exception as e:
            llm_duration = time.perf_counter() - llm_start

            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",  # Konsistenter Event-Typ
                    node_name="LLMToolNode",
                    session_id=prep_res.get("session_id"),
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=llm_duration,
                    llm_model=model_to_use,
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    },
                    metadata={"call_number": total_llm_calls + 1}
                ))
            eprint(f"LLM tool execution failed: {e}")
            final_response = f"I encountered an error while processing: {str(e)}"
            import traceback
            print(traceback.format_exc())
            break


    return {
        "success": True,
        "final_response": final_response or "I was unable to complete the request.",
        "tool_calls_made": tool_call_count,
        "conversation_history": conversation_history,
        "model_used": model_to_use,
        "llm_statistics": {
            "total_calls": total_llm_calls,
            "total_cost": total_cost,
            "total_tokens": total_tokens
        }
    }
PersonaConfig dataclass
Source code in toolboxv2/mods/isaa/base/Agent/types.py
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
@dataclass
class PersonaConfig:
    name: str
    style: str = "professional"
    personality_traits: list[str] = field(default_factory=lambda: ["helpful", "concise"])
    tone: str = "friendly"
    response_format: str = "direct"
    custom_instructions: str = ""

    format_config: FormatConfig  = None

    apply_method: str = "system_prompt"  # "system_prompt" | "post_process" | "both"
    integration_level: str = "light"  # "light" | "medium" | "heavy"

    def to_system_prompt_addition(self) -> str:
        """Convert persona to system prompt addition with format integration"""
        if self.apply_method in ["system_prompt", "both"]:
            additions = []
            additions.append(f"You are {self.name}.")
            additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

            if self.personality_traits:
                traits_str = ", ".join(self.personality_traits)
                additions.append(f"Your key traits are: {traits_str}.")

            if self.custom_instructions:
                additions.append(self.custom_instructions)

            # Format-spezifische Anweisungen hinzufügen
            if self.format_config:
                additions.append("\n" + self.format_config.get_combined_instructions())

            return " ".join(additions)
        return ""

    def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
        """Dynamische Format-Aktualisierung"""
        try:
            format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
            length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

            if not self.format_config:
                self.format_config = FormatConfig()

            self.format_config.response_format = format_enum
            self.format_config.text_length = length_enum

            if custom_instructions:
                self.format_config.custom_instructions = custom_instructions


        except ValueError:
            raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")

    def should_post_process(self) -> bool:
        """Check if post-processing should be applied"""
        return self.apply_method in ["post_process", "both"]
should_post_process()

Check if post-processing should be applied

Source code in toolboxv2/mods/isaa/base/Agent/types.py
764
765
766
def should_post_process(self) -> bool:
    """Check if post-processing should be applied"""
    return self.apply_method in ["post_process", "both"]
to_system_prompt_addition()

Convert persona to system prompt addition with format integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
def to_system_prompt_addition(self) -> str:
    """Convert persona to system prompt addition with format integration"""
    if self.apply_method in ["system_prompt", "both"]:
        additions = []
        additions.append(f"You are {self.name}.")
        additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

        if self.personality_traits:
            traits_str = ", ".join(self.personality_traits)
            additions.append(f"Your key traits are: {traits_str}.")

        if self.custom_instructions:
            additions.append(self.custom_instructions)

        # Format-spezifische Anweisungen hinzufügen
        if self.format_config:
            additions.append("\n" + self.format_config.get_combined_instructions())

        return " ".join(additions)
    return ""
update_format(response_format, text_length, custom_instructions='')

Dynamische Format-Aktualisierung

Source code in toolboxv2/mods/isaa/base/Agent/types.py
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
    """Dynamische Format-Aktualisierung"""
    try:
        format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
        length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

        if not self.format_config:
            self.format_config = FormatConfig()

        self.format_config.response_format = format_enum
        self.format_config.text_length = length_enum

        if custom_instructions:
            self.format_config.custom_instructions = custom_instructions


    except ValueError:
        raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")
PlanData

Bases: BaseModel

Dataclass for plan data

Source code in toolboxv2/mods/isaa/base/Agent/types.py
506
507
508
509
510
511
class PlanData(BaseModel):
    """Dataclass for plan data"""
    plan_name: str = Field(..., discription="Name of the plan")
    description: str = Field(..., discription="Description of the plan")
    execution_strategy: str = Field(..., discription="Execution strategy for the plan")
    tasks: list[LLMTask | ToolTask | DecisionTask] = Field(..., discription="List of tasks in the plan")
ProgressEvent dataclass

Enhanced progress event with better error handling

Source code in toolboxv2/mods/isaa/base/Agent/types.py
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
@dataclass
class ProgressEvent:

    """Enhanced progress event with better error handling"""

    # === 1. Kern-Attribute (Für jedes Event) ===
    event_type: str
    node_name: str
    timestamp: float = field(default_factory=time.time)
    event_id: str = field(default_factory=lambda: str(uuid.uuid4()))
    session_id: Optional[str] = None

    # === 2. Status und Ergebnis-Attribute ===
    status: Optional[NodeStatus] = None
    success: Optional[bool] = None
    duration: Optional[float] = None
    error_details: dict[str, Any] = field(default_factory=dict)  # Strukturiert: message, type, traceback

    # === 3. LLM-spezifische Attribute ===
    llm_model: Optional[str] = None
    llm_prompt_tokens: Optional[int] = None
    llm_completion_tokens: Optional[int] = None
    llm_total_tokens: Optional[int] = None
    llm_cost: Optional[float] = None
    llm_input: Optional[Any] = None  # Optional für Debugging, kann groß sein
    llm_output: Optional[str] = None # Optional für Debugging, kann groß sein

    # === 4. Tool-spezifische Attribute ===
    tool_name: Optional[str] = None
    is_meta_tool: Optional[bool] = None
    tool_args: Optional[dict[str, Any]] = None
    tool_result: Optional[Any] = None
    tool_error: Optional[str] = None
    llm_temperature: Optional[float]  = None

    # === 5. Strategie- und Kontext-Attribute ===
    agent_name: Optional[str] = None
    task_id: Optional[str] = None
    plan_id: Optional[str] = None


    # Node/Routing data
    routing_decision: Optional[str] = None
    node_phase: Optional[str] = None
    node_duration: Optional[float] = None

    # === 6. Metadaten (Für alles andere) ===
    metadata: dict[str, Any] = field(default_factory=dict)


    def __post_init__(self):

        if self.timestamp is None:
            self.timestamp = time.time()

        if self.metadata is None:
            self.metadata = {}
        if not self.event_id:
            self.event_id = f"{self.node_name}_{self.event_type}_{int(self.timestamp * 1000000)}"
        if 'error' in self.metadata or 'error_type' in self.metadata:
            if self.error_details is None:
                self.error_details = {}
            self.error_details['error'] = self.metadata.get('error')
            self.error_details['error_type'] = self.metadata.get('error_type')
            self.status = NodeStatus.FAILED
        if self.status == NodeStatus.FAILED:
            self.success = False
        if self.status == NodeStatus.COMPLETED:
            self.success = True

    def _to_dict(self) -> dict[str, Any]:
        """Convert ProgressEvent to dictionary with proper handling of all field types"""
        result = {}

        # Get all fields from the dataclass
        for field in fields(self):
            value = getattr(self, field.name)

            # Handle None values
            if value is None:
                result[field.name] = None
                continue

            # Handle NodeStatus enum
            if isinstance(value, NodeStatus | Enum):
                result[field.name] = value.value
            # Handle dataclass objects
            elif is_dataclass(value):
                result[field.name] = asdict(value)
            # Handle dictionaries (recursively process nested enums/dataclasses)
            elif isinstance(value, dict):
                result[field.name] = self._process_dict(value)
            # Handle lists (recursively process nested items)
            elif isinstance(value, list):
                result[field.name] = self._process_list(value)
            # Handle primitive types
            else:
                result[field.name] = value

        return result

    def _process_dict(self, d: dict[str, Any]) -> dict[str, Any]:
        """Recursively process dictionary values"""
        result = {}
        for k, v in d.items():
            if isinstance(v, Enum):
                result[k] = v.value
            elif is_dataclass(v):
                result[k] = asdict(v)
            elif isinstance(v, dict):
                result[k] = self._process_dict(v)
            elif isinstance(v, list):
                result[k] = self._process_list(v)
            else:
                result[k] = v
        return result

    def _process_list(self, lst: list[Any]) -> list[Any]:
        """Recursively process list items"""
        result = []
        for item in lst:
            if isinstance(item, Enum):
                result.append(item.value)
            elif is_dataclass(item):
                result.append(asdict(item))
            elif isinstance(item, dict):
                result.append(self._process_dict(item))
            elif isinstance(item, list):
                result.append(self._process_list(item))
            else:
                result.append(item)
        return result

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
        """Create ProgressEvent from dictionary"""
        # Create a copy to avoid modifying the original
        data_copy = dict(data)

        # Handle NodeStatus enum conversion from string back to enum
        if 'status' in data_copy and data_copy['status'] is not None:
            if isinstance(data_copy['status'], str):
                try:
                    data_copy['status'] = NodeStatus(data_copy['status'])
                except (ValueError, TypeError):
                    # If invalid status value, set to None
                    data_copy['status'] = None

        # Filter out any keys that aren't valid dataclass fields
        field_names = {field.name for field in fields(cls)}
        filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

        # Ensure metadata is properly initialized
        if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
            filtered_data['metadata'] = {}

        return cls(**filtered_data)

    def to_dict(self) -> dict[str, Any]:
        """Return event data with None values removed for compact display"""
        data = self._to_dict()

        def clean_dict(d):
            if isinstance(d, dict):
                return {k: clean_dict(v) for k, v in d.items()
                        if v is not None and v != {} and v != [] and v != ''}
            elif isinstance(d, list):
                cleaned_list = [clean_dict(item) for item in d if item is not None]
                return [item for item in cleaned_list if item != {} and item != []]
            return d

        return clean_dict(data)

    def get_chat_display_data(self) -> dict[str, Any]:
        """Get data optimized for chat view display"""
        filtered = self.filter_none_values()

        # Core fields always shown
        core_data = {
            'event_type': filtered.get('event_type'),
            'node_name': filtered.get('node_name'),
            'timestamp': filtered.get('timestamp'),
            'event_id': filtered.get('event_id'),
            'status': filtered.get('status')
        }

        # Add specific fields based on event type
        if self.event_type == 'outline_created':
            if 'metadata' in filtered:
                core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
        elif self.event_type == 'reasoning_loop':
            if 'metadata' in filtered:
                core_data.update({
                    'loop_number': filtered['metadata'].get('loop_number'),
                    'outline_step': filtered['metadata'].get('outline_step'),
                    'context_size': filtered['metadata'].get('context_size')
                })
        elif self.event_type == 'tool_call':
            core_data.update({
                'tool_name': filtered.get('tool_name'),
                'is_meta_tool': filtered.get('is_meta_tool')
            })
        elif self.event_type == 'llm_call':
            core_data.update({
                'llm_model': filtered.get('llm_model'),
                'llm_total_tokens': filtered.get('llm_total_tokens'),
                'llm_cost': filtered.get('llm_cost')
            })

        # Remove None values from core_data
        return {k: v for k, v in core_data.items() if v is not None}

    def get_detailed_display_data(self) -> dict[str, Any]:
        """Get complete filtered data for detailed popup view"""
        return self.filter_none_values()

    def get_progress_summary(self) -> str:
        """Get a brief summary for progress sidebar"""
        if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
            metadata = self.filter_none_values()['metadata']
            loop_num = metadata.get('loop_number', '?')
            step = metadata.get('outline_step', '?')
            return f"Loop {loop_num}, Step {step}"
        elif self.event_type == 'tool_call':
            tool_name = self.tool_name or 'Unknown Tool'
            return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
        elif self.event_type == 'llm_call':
            model = self.llm_model or 'Unknown Model'
            tokens = self.llm_total_tokens
            return f"{model} ({tokens} tokens)" if tokens else model
        else:
            return self.event_type.replace('_', ' ').title()
from_dict(data) classmethod

Create ProgressEvent from dictionary

Source code in toolboxv2/mods/isaa/base/Agent/types.py
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
    """Create ProgressEvent from dictionary"""
    # Create a copy to avoid modifying the original
    data_copy = dict(data)

    # Handle NodeStatus enum conversion from string back to enum
    if 'status' in data_copy and data_copy['status'] is not None:
        if isinstance(data_copy['status'], str):
            try:
                data_copy['status'] = NodeStatus(data_copy['status'])
            except (ValueError, TypeError):
                # If invalid status value, set to None
                data_copy['status'] = None

    # Filter out any keys that aren't valid dataclass fields
    field_names = {field.name for field in fields(cls)}
    filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

    # Ensure metadata is properly initialized
    if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
        filtered_data['metadata'] = {}

    return cls(**filtered_data)
get_chat_display_data()

Get data optimized for chat view display

Source code in toolboxv2/mods/isaa/base/Agent/types.py
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
def get_chat_display_data(self) -> dict[str, Any]:
    """Get data optimized for chat view display"""
    filtered = self.filter_none_values()

    # Core fields always shown
    core_data = {
        'event_type': filtered.get('event_type'),
        'node_name': filtered.get('node_name'),
        'timestamp': filtered.get('timestamp'),
        'event_id': filtered.get('event_id'),
        'status': filtered.get('status')
    }

    # Add specific fields based on event type
    if self.event_type == 'outline_created':
        if 'metadata' in filtered:
            core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
    elif self.event_type == 'reasoning_loop':
        if 'metadata' in filtered:
            core_data.update({
                'loop_number': filtered['metadata'].get('loop_number'),
                'outline_step': filtered['metadata'].get('outline_step'),
                'context_size': filtered['metadata'].get('context_size')
            })
    elif self.event_type == 'tool_call':
        core_data.update({
            'tool_name': filtered.get('tool_name'),
            'is_meta_tool': filtered.get('is_meta_tool')
        })
    elif self.event_type == 'llm_call':
        core_data.update({
            'llm_model': filtered.get('llm_model'),
            'llm_total_tokens': filtered.get('llm_total_tokens'),
            'llm_cost': filtered.get('llm_cost')
        })

    # Remove None values from core_data
    return {k: v for k, v in core_data.items() if v is not None}
get_detailed_display_data()

Get complete filtered data for detailed popup view

Source code in toolboxv2/mods/isaa/base/Agent/types.py
263
264
265
def get_detailed_display_data(self) -> dict[str, Any]:
    """Get complete filtered data for detailed popup view"""
    return self.filter_none_values()
get_progress_summary()

Get a brief summary for progress sidebar

Source code in toolboxv2/mods/isaa/base/Agent/types.py
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
def get_progress_summary(self) -> str:
    """Get a brief summary for progress sidebar"""
    if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
        metadata = self.filter_none_values()['metadata']
        loop_num = metadata.get('loop_number', '?')
        step = metadata.get('outline_step', '?')
        return f"Loop {loop_num}, Step {step}"
    elif self.event_type == 'tool_call':
        tool_name = self.tool_name or 'Unknown Tool'
        return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
    elif self.event_type == 'llm_call':
        model = self.llm_model or 'Unknown Model'
        tokens = self.llm_total_tokens
        return f"{model} ({tokens} tokens)" if tokens else model
    else:
        return self.event_type.replace('_', ' ').title()
to_dict()

Return event data with None values removed for compact display

Source code in toolboxv2/mods/isaa/base/Agent/types.py
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def to_dict(self) -> dict[str, Any]:
    """Return event data with None values removed for compact display"""
    data = self._to_dict()

    def clean_dict(d):
        if isinstance(d, dict):
            return {k: clean_dict(v) for k, v in d.items()
                    if v is not None and v != {} and v != [] and v != ''}
        elif isinstance(d, list):
            cleaned_list = [clean_dict(item) for item in d if item is not None]
            return [item for item in cleaned_list if item != {} and item != []]
        return d

    return clean_dict(data)
ProgressTracker

Advanced progress tracking with cost calculation

Source code in toolboxv2/mods/isaa/base/Agent/types.py
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
class ProgressTracker:
    """Advanced progress tracking with cost calculation"""

    def __init__(self, progress_callback: callable  = None, agent_name="unknown"):
        self.progress_callback = progress_callback
        self.events: list[ProgressEvent] = []
        self.active_timers: dict[str, float] = {}

        # Cost tracking (simplified - would need actual provider pricing)
        self.token_costs = {
            "input": 0.00001,  # $0.01/1K tokens input
            "output": 0.00003,  # $0.03/1K tokens output
        }
        self.agent_name = agent_name

    async def emit_event(self, event: ProgressEvent):
        """Emit progress event with callback and storage"""
        self.events.append(event)
        event.agent_name = self.agent_name

        if self.progress_callback:
            try:
                if asyncio.iscoroutinefunction(self.progress_callback):
                    await self.progress_callback(event)
                else:
                    self.progress_callback(event)
            except Exception:
                import traceback
                print(traceback.format_exc())


    def start_timer(self, key: str) -> float:
        """Start timing operation"""
        start_time = time.perf_counter()
        self.active_timers[key] = start_time
        return start_time

    def end_timer(self, key: str) -> float:
        """End timing operation and return duration"""
        if key not in self.active_timers:
            return 0.0
        duration = time.perf_counter() - self.active_timers[key]
        del self.active_timers[key]
        return duration

    def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
        """Calculate approximate LLM cost"""
        cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
        if hasattr(completion_response, "_hidden_params"):
            cost = completion_response._hidden_params.get("response_cost", 0)
        try:
            import litellm
            cost = litellm.completion_cost(model=model, completion_response=completion_response)
        except ImportError:
            pass
        except Exception as e:
            try:
                import litellm
                cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
            except Exception:
                pass
        return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]

    def get_summary(self) -> dict[str, Any]:
        """Get comprehensive progress summary"""
        summary = {
            "total_events": len(self.events),
            "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
            "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
            "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
            "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
            "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
            "nodes_visited": list(set(e.node_name for e in self.events)),
            "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
            "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
        }
        return summary
calculate_llm_cost(model, input_tokens, output_tokens, completion_response=None)

Calculate approximate LLM cost

Source code in toolboxv2/mods/isaa/base/Agent/types.py
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
    """Calculate approximate LLM cost"""
    cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
    if hasattr(completion_response, "_hidden_params"):
        cost = completion_response._hidden_params.get("response_cost", 0)
    try:
        import litellm
        cost = litellm.completion_cost(model=model, completion_response=completion_response)
    except ImportError:
        pass
    except Exception as e:
        try:
            import litellm
            cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
        except Exception:
            pass
    return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
emit_event(event) async

Emit progress event with callback and storage

Source code in toolboxv2/mods/isaa/base/Agent/types.py
299
300
301
302
303
304
305
306
307
308
309
310
311
312
async def emit_event(self, event: ProgressEvent):
    """Emit progress event with callback and storage"""
    self.events.append(event)
    event.agent_name = self.agent_name

    if self.progress_callback:
        try:
            if asyncio.iscoroutinefunction(self.progress_callback):
                await self.progress_callback(event)
            else:
                self.progress_callback(event)
        except Exception:
            import traceback
            print(traceback.format_exc())
end_timer(key)

End timing operation and return duration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
321
322
323
324
325
326
327
def end_timer(self, key: str) -> float:
    """End timing operation and return duration"""
    if key not in self.active_timers:
        return 0.0
    duration = time.perf_counter() - self.active_timers[key]
    del self.active_timers[key]
    return duration
get_summary()

Get comprehensive progress summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py
347
348
349
350
351
352
353
354
355
356
357
358
359
360
def get_summary(self) -> dict[str, Any]:
    """Get comprehensive progress summary"""
    summary = {
        "total_events": len(self.events),
        "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
        "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
        "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
        "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
        "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
        "nodes_visited": list(set(e.node_name for e in self.events)),
        "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
        "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
    }
    return summary
start_timer(key)

Start timing operation

Source code in toolboxv2/mods/isaa/base/Agent/types.py
315
316
317
318
319
def start_timer(self, key: str) -> float:
    """Start timing operation"""
    start_time = time.perf_counter()
    self.active_timers[key] = start_time
    return start_time
ResponseFinalProcessorNode

Bases: AsyncNode

Finale Verarbeitung mit Persona-System

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
@with_progress_tracking
class ResponseFinalProcessorNode(AsyncNode):
    """Finale Verarbeitung mit Persona-System"""

    async def prep_async(self, shared):
        return {
            "formatted_response": shared.get("formatted_response", {}),
            "quality_assessment": shared.get("quality_assessment", {}),
            "conversation_history": shared.get("conversation_history", []),
            "persona": shared.get("persona_config"),
            "fast_llm_model": shared.get("fast_llm_model"),
            "use_fast_response": shared.get("use_fast_response", True),
            "agent_instance": shared.get("agent_instance"),
        }

    async def exec_async(self, prep_res):
        response_data = prep_res["formatted_response"]
        raw_response = response_data.get("formatted_response", "I apologize, but I couldn't generate a response.")

        # Persona-basierte Anpassung
        if prep_res.get("persona") and LITELLM_AVAILABLE:
            final_response = await self._apply_persona_style(raw_response, prep_res)
        else:
            final_response = raw_response

        # Finale Metadaten
        processing_metadata = {
            "response_confidence": response_data.get("confidence", 0.0),
            "quality_score": prep_res.get("quality_assessment", {}).get("quality_score", 0.0),
            "processing_timestamp": datetime.now().isoformat(),
            "response_length": len(final_response),
            "persona_applied": prep_res.get("persona") is not None
        }

        return {
            "final_response": final_response,
            "metadata": processing_metadata,
            "status": "completed"
        }

    async def _apply_persona_style(self, response: str, prep_res: dict) -> str:
        """Optimized persona styling mit Konfiguration"""
        persona = prep_res["persona"]

        # Nur anwenden wenn post-processing konfiguriert
        if not persona.should_post_process():
            return response

        # Je nach Integration Level unterschiedliche Prompts
        if persona.integration_level == "light":
            style_prompt = f"Make this {persona.tone} and {persona.style}: {response}"
            max_tokens = 400
        elif persona.integration_level == "medium":
            style_prompt = f"""
    Apply {persona.name} persona (style: {persona.style}, tone: {persona.tone}) to:
    {response}

    Keep the same information, adjust presentation:"""
            max_tokens = 600
        else:  # heavy
            style_prompt = f"""
Completely transform as {persona.name}:
Style: {persona.style}, Tone: {persona.tone}
Traits: {', '.join(persona.personality_traits)}
Instructions: {persona.custom_instructions}

Original: {response}

As {persona.name}:"""
            max_tokens = 1000

        try:
            model_to_use = prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")
            agent_instance = prep_res["agent_instance"]
            if prep_res.get("use_fast_response", True):
                response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=[{"role": "user", "content": style_prompt}],
                    temperature=0.5,
                    max_tokens=max_tokens, node_name="PersonaStylingNode", task_id="persona_styling_fast"
                )
            else:
                response = await agent_instance.a_run_llm_completion(
                    model=model_to_use,
                    messages=[{"role": "user", "content": style_prompt}],
                    temperature=0.6,
                    max_tokens=max_tokens + 200, node_name="PersonaStylingNode", task_id="persona_styling_ritch"
                )

            return response.strip()

        except Exception as e:
            wprint(f"Persona styling failed: {e}")
            return response

    async def post_async(self, shared, prep_res, exec_res):
        shared["current_response"] = exec_res["final_response"]
        shared["response_metadata"] = exec_res["metadata"]
        return "response_ready"
ResponseFormatterNode

Bases: AsyncNode

Formatiere finale Antwort für Benutzer

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
@with_progress_tracking
class ResponseFormatterNode(AsyncNode):
    """Formatiere finale Antwort für Benutzer"""

    async def prep_async(self, shared):
        return {
            "synthesized_response": shared.get("synthesized_response", {}),
            "original_query": shared.get("current_query", ""),
            "user_preferences": shared.get("user_preferences", {})
        }

    async def exec_async(self, prep_res):
        synthesis_data = prep_res["synthesized_response"]
        raw_response = synthesis_data.get("synthesized_response", "")

        if not raw_response:
            return {
                "formatted_response": "I apologize, but I was unable to generate a meaningful response to your query."}

        # Basis-Formatierung
        formatted_response = raw_response.strip()

        # Füge Metadaten hinzu falls gewünscht (für debugging/transparency)
        confidence = synthesis_data.get("confidence", 0.0)
        if confidence < 0.4:
            formatted_response += "\n\n*Note: This response has low confidence due to limited information.*"

        adaptation_note = ""
        synthesis_method = synthesis_data.get("synthesis_method", "unknown")
        if synthesis_method == "fallback":
            adaptation_note = "\n\n*Note: Response generated with limited processing capabilities.*"

        return {
            "formatted_response": formatted_response + adaptation_note,
            "confidence": confidence,
            "metadata": {
                "synthesis_method": synthesis_method,
                "response_length": len(formatted_response)
            }
        }

    async def post_async(self, shared, prep_res, exec_res):
        shared["formatted_response"] = exec_res
        return "formatted"
ResponseGenerationFlow

Bases: AsyncFlow

Intelligente Antwortgenerierung basierend auf Task-Ergebnissen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
@with_progress_tracking
class ResponseGenerationFlow(AsyncFlow):
    """Intelligente Antwortgenerierung basierend auf Task-Ergebnissen"""

    def __init__(self, tools=None):
        # Nodes für Response-Pipeline
        self.context_aggregator = ContextAggregatorNode()
        self.result_synthesizer = ResultSynthesizerNode()
        self.response_formatter = ResponseFormatterNode()
        self.quality_checker = ResponseQualityNode()
        self.final_processor = ResponseFinalProcessorNode()

        # === RESPONSE GENERATION PIPELINE ===

        # Context Aggregation -> Synthesis
        self.context_aggregator - "context_ready" >> self.result_synthesizer
        self.context_aggregator - "no_context" >> self.response_formatter  # Fallback

        # Synthesis -> Formatting
        self.result_synthesizer - "synthesized" >> self.response_formatter
        self.result_synthesizer - "synthesis_failed" >> self.response_formatter

        # Formatting -> Quality Check
        self.response_formatter - "formatted" >> self.quality_checker
        self.response_formatter - "format_failed" >> self.final_processor  # Skip quality check

        # Quality Check -> Final Processing oder Retry
        self.quality_checker - "quality_good" >> self.final_processor
        self.quality_checker - "quality_poor" >> self.result_synthesizer  # Retry synthesis
        self.quality_checker - "quality_acceptable" >> self.final_processor

        super().__init__(start=self.context_aggregator)
ResponseQualityNode

Bases: AsyncNode

Prüfe Qualität der generierten Antwort

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
@with_progress_tracking
class ResponseQualityNode(AsyncNode):
    """Prüfe Qualität der generierten Antwort"""

    async def prep_async(self, shared):
        return {
            "formatted_response": shared.get("formatted_response", {}),
            "original_query": shared.get("current_query", ""),
            "format_config": self._get_format_config(shared),
            "fast_llm_model": shared.get("fast_llm_model"),
            "persona_config": shared.get("persona_config"),
            "agent_instance": shared.get("agent_instance"),
        }

    def _get_format_config(self, shared) -> FormatConfig | None:
        """Extrahiere Format-Konfiguration"""
        persona = shared.get("persona_config")
        if persona and hasattr(persona, 'format_config'):
            return persona.format_config
        return None

    async def exec_async(self, prep_res):
        response_data = prep_res["formatted_response"]
        response_text = response_data.get("formatted_response", "")
        original_query = prep_res["original_query"]
        format_config = prep_res["format_config"]

        # Basis-Qualitätsprüfung
        base_quality = self._heuristic_quality_check(response_text, original_query)

        # Format-spezifische Bewertung
        format_quality = await self._evaluate_format_adherence(response_text, format_config)

        # Längen-spezifische Bewertung
        length_quality = self._evaluate_length_adherence(response_text, format_config)

        # LLM-basierte Gesamtbewertung
        llm_quality = 0.5
        if LITELLM_AVAILABLE and len(response_text) > 500:
            llm_quality = await self._llm_format_quality_check(
                response_text, original_query, format_config, prep_res
            )

        # Gewichtete Gesamtbewertung
        total_quality = (
            base_quality * 0.3 +
            format_quality * 0.3 +
            length_quality * 0.2 +
            llm_quality * 0.2
        )

        quality_details = {
            "total_score": total_quality,
            "base_quality": base_quality,
            "format_adherence": format_quality,
            "length_adherence": length_quality,
            "llm_assessment": llm_quality,
            "format_config_used": format_config is not None
        }

        return {
            "quality_score": total_quality,
            "quality_assessment": self._score_to_assessment(total_quality),
            "quality_details": quality_details,
            "suggestions": self._generate_format_quality_suggestions(
                total_quality, response_text, format_config, quality_details
            )
        }

    async def _evaluate_format_adherence(self, response: str, format_config: FormatConfig | None) -> float:
        """Bewerte Format-Einhaltung"""
        if not format_config:
            return 0.8  # Neutral wenn kein Format vorgegeben

        format_type = format_config.response_format
        score = 0.5

        # Format-spezifische Checks
        if format_type == ResponseFormat.WITH_TABLES:
            if '|' in response or 'Table:' in response or '| ' in response:
                score += 0.4

        elif format_type == ResponseFormat.WITH_BULLET_POINTS:
            bullet_count = response.count('•') + response.count('-') + response.count('*')
            if bullet_count >= 2:
                score += 0.4
            elif bullet_count >= 1:
                score += 0.2

        elif format_type == ResponseFormat.WITH_LISTS:
            list_patterns = ['1.', '2.', '3.', 'a)', 'b)', 'c)']
            list_score = sum(1 for pattern in list_patterns if pattern in response)
            score += min(0.4, list_score * 0.1)

        elif format_type == ResponseFormat.MD_TEXT:
            md_elements = ['#', '**', '*', '`', '```', '[', ']', '(', ')']
            md_score = sum(1 for element in md_elements if element in response)
            score += min(0.4, md_score * 0.05)

        elif format_type == ResponseFormat.YAML_TEXT:
            if response.strip().startswith(('```yaml', '---')) or ': ' in response:
                score += 0.4

        elif format_type == ResponseFormat.JSON_TEXT:
            if response.strip().startswith(('{', '[')):
                try:
                    json.loads(response)
                    score += 0.4
                except:
                    score += 0.1  # Partial credit for JSON-like structure

        elif format_type == ResponseFormat.TEXT_ONLY:
            # Penalize if formatting elements are present
            format_elements = ['#', '*', '|', '```', '1.', '•', '-']
            format_count = sum(1 for element in format_elements if element in response)
            score += max(0.1, 0.5 - format_count * 0.05)

        elif format_type == ResponseFormat.PSEUDO_CODE:
            code_indicators = ['if ', 'for ', 'while ', 'def ', 'return ', 'function', 'BEGIN', 'END']
            code_score = sum(1 for indicator in code_indicators if indicator in response)
            score += min(0.4, code_score * 0.1)

        return max(0.0, min(1.0, score))

    def _evaluate_length_adherence(self, response: str, format_config: FormatConfig | None) -> float:
        """Bewerte Längen-Einhaltung"""
        if not format_config:
            return 0.8

        word_count = len(response.split())
        min_words, max_words = format_config.get_expected_word_range()

        if min_words <= word_count <= max_words:
            return 1.0
        elif word_count < min_words:
            # Zu kurz - sanfte Bestrafung
            ratio = word_count / min_words
            return max(0.3, ratio * 0.8)
        else:  # word_count > max_words
            # Zu lang - weniger Bestrafung als zu kurz
            excess_ratio = (word_count - max_words) / max_words
            return max(0.4, 1.0 - excess_ratio * 0.3)

    async def _llm_format_quality_check(
        self,
        response: str,
        query: str,
        format_config: FormatConfig | None,
        prep_res: dict
    ) -> float:
        """LLM-basierte Format- und Qualitätsbewertung"""
        if not format_config:
            return await self._standard_llm_quality_check(response, query, prep_res)

        format_desc = format_config.get_format_instructions()
        length_desc = format_config.get_length_instructions()

        prompt = f"""
Bewerte diese Antwort auf einer Skala von 0.0 bis 1.0 basierend auf Format-Einhaltung und Qualität:

Benutzer-Anfrage: {query}

Antwort: {response}

Erwartetes Format: {format_desc}
Erwartete Länge: {length_desc}

Bewertungskriterien:
1. Format-Einhaltung (40%): Entspricht die Antwort dem geforderten Format?
2. Längen-Angemessenheit (25%): Ist die Länge angemessen?
3. Inhaltliche Qualität (25%): Beantwortet die Anfrage vollständig?
4. Lesbarkeit und Struktur (10%): Ist die Antwort gut strukturiert?

Antworte nur mit einer Zahl zwischen 0.0 und 1.0:"""

        try:
            model_to_use = prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")
            agent_instance = prep_res["agent_instance"]
            score_text = (await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.1,
                max_tokens=10,
                node_name="QualityAssessmentNode", task_id="format_quality_assessment"
            )).strip()

            return float(score_text)

        except Exception as e:
            wprint(f"LLM format quality check failed: {e}")
            return 0.6  # Neutral fallback

    def _generate_format_quality_suggestions(
        self,
        score: float,
        response: str,
        format_config: FormatConfig | None,
        quality_details: dict
    ) -> list[str]:
        """Generiere Format-spezifische Verbesserungsvorschläge"""
        suggestions = []

        if not format_config:
            return ["Consider defining a specific response format for better consistency"]

        # Format-spezifische Vorschläge
        if quality_details["format_adherence"] < 0.6:
            format_type = format_config.response_format

            if format_type == ResponseFormat.WITH_TABLES:
                suggestions.append("Add tables using markdown format (| Column | Column |)")
            elif format_type == ResponseFormat.WITH_BULLET_POINTS:
                suggestions.append("Use bullet points (•, -, *) to structure information")
            elif format_type == ResponseFormat.MD_TEXT:
                suggestions.append("Use markdown formatting (headers, bold, code blocks)")
            elif format_type == ResponseFormat.YAML_TEXT:
                suggestions.append("Format response as valid YAML structure")
            elif format_type == ResponseFormat.JSON_TEXT:
                suggestions.append("Format response as valid JSON")

        # Längen-spezifische Vorschläge
        if quality_details["length_adherence"] < 0.6:
            word_count = len(response.split())
            min_words, max_words = format_config.get_expected_word_range()

            if word_count < min_words:
                suggestions.append(f"Response too short ({word_count} words). Aim for {min_words}-{max_words} words")
            else:
                suggestions.append(f"Response too long ({word_count} words). Aim for {min_words}-{max_words} words")

        # Qualitäts-spezifische Vorschläge
        if score < 0.5:
            suggestions.append("Overall quality needs improvement - consider regenerating")
        elif score < 0.7:
            suggestions.append("Good response but could be enhanced with better format adherence")

        return suggestions

    async def _standard_llm_quality_check(self, response: str, query: str, prep_res: dict) -> float:
        """Standard LLM-Qualitätsprüfung ohne Format-Fokus"""
        # Bestehende Implementierung beibehalten
        return await self._llm_quality_check(response, query, prep_res)

    def _heuristic_quality_check(self, response: str, query: str) -> float:
        """Heuristische Qualitätsprüfung"""
        score = 0.5  # Base score

        # Length check
        if len(response) < 50:
            score -= 0.3
        elif len(response) > 100:
            score += 0.2

        # Query term coverage
        query_terms = set(query.lower().split())
        response_terms = set(response.lower().split())
        coverage = len(query_terms.intersection(response_terms)) / max(len(query_terms), 1)
        score += coverage * 0.3

        # Structure indicators
        if any(indicator in response for indicator in [":", "-", "1.", "•"]):
            score += 0.1  # Structured response bonus

        return max(0.0, min(1.0, score))

    async def _llm_quality_check(self, response: str, query: str, prep_res: dict) -> float:
        """LLM-basierte Qualitätsprüfung"""
        try:
            prompt = f"""
Rate the quality of this response to the user's query on a scale of 0.0 to 1.0.

User Query: {query}

Response: {response}

Consider:
- Relevance to the query
- Completeness of information
- Clarity and readability
- Accuracy (if verifiable)

Respond with just a number between 0.0 and 1.0:"""

            model_to_use = prep_res.get("fast_llm_model", "openrouter/anthropic/claude-3-haiku")
            agent_instance = prep_res["agent_instance"]
            score_text = (await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.1,
                max_tokens=10,
                node_name="QualityAssessmentNode", task_id="quality_assessment"
            )).strip()

            return float(score_text)

        except:
            return 0.5  # Fallback score

    def _score_to_assessment(self, score: float) -> str:
        if score >= 0.8:
            return "quality_good"
        elif score >= 0.5:
            return "quality_acceptable"
        else:
            return "quality_poor"

    async def post_async(self, shared, prep_res, exec_res):
        shared["quality_assessment"] = exec_res
        return exec_res["quality_assessment"]
ResultSynthesizerNode

Bases: AsyncNode

Synthetisiere finale Antwort aus allen Ergebnissen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
@with_progress_tracking
class ResultSynthesizerNode(AsyncNode):
    """Synthetisiere finale Antwort aus allen Ergebnissen"""

    async def prep_async(self, shared):
        return {
            "aggregated_context": shared.get("aggregated_context", {}),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "agent_instance": shared.get("agent_instance")
        }

    async def exec_async(self, prep_res):
        if not LITELLM_AVAILABLE:
            return await self._fallback_synthesis(prep_res)

        context = prep_res["aggregated_context"]
        persona = (prep_res['agent_instance'].amd.persona.to_system_prompt_addition() if not prep_res['agent_instance'].amd.persona.should_post_process() else '') if prep_res['agent_instance'].amd.persona else None
        prompt = f"""
Du bist ein Experte für Informationssynthese. Erstelle eine umfassende, hilfreiche Antwort basierend auf den gesammelten Ergebnissen.

## Ursprüngliche Anfrage
{context.get('original_query', '')}

## Erfolgreiche Ergebnisse
{self._format_successful_results(context.get('successful_results', {}))}

## Wichtige Entdeckungen
{self._format_key_discoveries(context.get('key_discoveries', []))}

## Plan-Adaptationen
{context.get('adaptation_summary', 'No adaptations were needed.')}

## Fehlgeschlagene Versuche
{self._format_failed_attempts(context.get('failed_attempts', {}))}

{persona}

## Anweisungen
1. Gib eine direkte, hilfreiche Antwort auf die ursprüngliche Anfrage
2. Integriere alle relevanten gefundenen Informationen
3. Erkläre kurz den Prozess, falls Adaptationen nötig waren
4. Sei ehrlich über Limitationen oder fehlende Informationen
5. Strukturiere die Antwort logisch und lesbar

Erstelle eine finale Antwort:"""

        try:
            # Verwende complex model für finale Synthesis
            model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")
            agent_instance = prep_res["agent_instance"]
            synthesized_response = await agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.3,
                max_tokens=1500,
                node_name="ResultSynthesizerNode", task_id="response_synthesis"
            )

            return {
                "synthesized_response": synthesized_response,
                "synthesis_method": "llm",
                "model_used": model_to_use,
                "confidence": self._estimate_synthesis_confidence(context)
            }

        except Exception as e:
            eprint(f"LLM synthesis failed: {e}")
            return await self._fallback_synthesis(prep_res)

    def _format_successful_results(self, results: dict) -> str:
        formatted = []
        for _task_id, result_info in results.items():
            formatted.append(f"- {result_info['task_description']}: {str(result_info['result'])[:20000]}...")
        return "\n".join(formatted) if formatted else "No successful results to report."

    def _format_key_discoveries(self, discoveries: list) -> str:
        formatted = []
        for discovery in discoveries:
            confidence = discovery.get('confidence', 0.0)
            formatted.append(f"- {discovery['discovery']} (Confidence: {confidence:.2f})")
        return "\n".join(formatted) if formatted else "No key discoveries."

    def _format_failed_attempts(self, failed: dict) -> str:
        if not failed:
            return "No significant failures."
        formatted = [f"- {info['description']}: {info['error']}" for info in failed.values()]
        return "\n".join(formatted)

    async def _fallback_synthesis(self, prep_res) -> dict:
        """Fallback synthesis ohne LLM"""
        context = prep_res["aggregated_context"]

        # Einfache Template-basierte Synthese
        response_parts = []

        if context.get("key_discoveries"):
            response_parts.append("Based on my analysis, I found:")
            for discovery in context["key_discoveries"][:3]:  # Top 3
                response_parts.append(f"- {discovery['discovery']}")

        if context.get("successful_results"):
            response_parts.append("\nDetailed results:")
            for _task_id, result in list(context["successful_results"].items())[:2]:  # Top 2
                response_parts.append(f"- {result['task_description']}: {str(result['result'])[:150]}")

        if context.get("adaptation_summary"):
            response_parts.append(f"\n{context['adaptation_summary']}")

        fallback_response = "\n".join(
            response_parts) if response_parts else "I was unable to complete the requested task effectively."

        return {
            "synthesized_response": fallback_response,
            "synthesis_method": "fallback",
            "confidence": 0.3
        }

    def _estimate_synthesis_confidence(self, context: dict) -> float:
        """Schätze Confidence der Synthese"""
        confidence = 0.5  # Base confidence

        # Boost für erfolgreiche Ergebnisse
        successful_count = len(context.get("successful_results", {}))
        confidence += min(successful_count * 0.15, 0.3)

        # Boost für key discoveries mit hoher confidence
        for discovery in context.get("key_discoveries", []):
            discovery_conf = discovery.get("confidence", 0.0)
            confidence += discovery_conf * 0.1

        # Penalty für viele fehlgeschlagene Versuche
        failed_count = len(context.get("failed_attempts", {}))
        confidence -= min(failed_count * 0.1, 0.2)

        return max(0.1, min(1.0, confidence))

    async def post_async(self, shared, prep_res, exec_res):
        shared["synthesized_response"] = exec_res
        if exec_res.get("synthesized_response"):
            return "synthesized"
        else:
            return "synthesis_failed"
StateSyncNode

Bases: AsyncNode

Synchronize state between world model and shared store

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
@with_progress_tracking
class StateSyncNode(AsyncNode):
    """Synchronize state between world model and shared store"""
    async def prep_async(self, shared):
        world_model = shared.get("world_model", {})
        session_data = shared.get("session_data", {})
        tasks = shared.get("tasks", {})
        system_status = shared.get("system_status", "idle")

        return {
            "world_model": world_model,
            "session_data": session_data,
            "tasks": tasks,
            "system_status": system_status,
            "sync_timestamp": datetime.now().isoformat()
        }

    async def exec_async(self, prep_res):
        # Perform intelligent state synchronization
        sync_result = {
            "world_model_updates": {},
            "session_updates": {},
            "task_updates": {},
            "conflicts_resolved": [],
            "sync_successful": True
        }

        # Update world model with new information
        if "current_response" in prep_res:
            # Extract learnable facts from responses
            extracted_facts = self._extract_facts(prep_res.get("current_response", ""))
            sync_result["world_model_updates"].update(extracted_facts)

        # Sync task states
        for task_id, task in prep_res["tasks"].items():
            if task.status == "completed" and task.result:
                # Store task results in world model
                fact_key = f"task_{task_id}_result"
                sync_result["world_model_updates"][fact_key] = task.result

        return sync_result

    def _extract_facts(self, text: str) -> dict[str, Any]:
        """Extract learnable facts from text"""
        facts = {}
        lines = text.split('\n')

        for line in lines:
            line = line.strip()
            # Look for definitive statements
            if ' is ' in line and not line.startswith('I ') and not '?' in line:
                parts = line.split(' is ', 1)
                if len(parts) == 2:
                    subject = parts[0].strip().lower()
                    predicate = parts[1].strip().rstrip('.')
                    if len(subject.split()) <= 3:  # Keep subjects simple
                        facts[subject] = predicate

        return facts

    async def post_async(self, shared, prep_res, exec_res):
        # Apply the synchronization results
        if exec_res["sync_successful"]:
            shared["world_model"].update(exec_res["world_model_updates"])
            shared["session_data"].update(exec_res["session_updates"])
            shared["last_sync"] = datetime.now()
            return "sync_complete"
        else:
            wprint("State synchronization failed")
            return "sync_failed"
Task dataclass
Source code in toolboxv2/mods/isaa/base/Agent/types.py
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
@dataclass
class Task:
    id: str
    type: str
    description: str
    status: str = "pending"  # pending, running, completed, failed, paused
    priority: int = 1
    dependencies: list[str] = field(default_factory=list)
    subtasks: list[str] = field(default_factory=list)
    result: Any = None
    error: str = None
    created_at: datetime = field(default_factory=datetime.now)
    started_at: datetime  = None
    completed_at: datetime  = None
    metadata: dict[str, Any] = field(default_factory=dict)
    retry_count: int = 0
    max_retries: int = 3
    critical: bool = False

    task_identification_attr: bool = True


    def __post_init__(self):
        """Ensure all mutable defaults are properly initialized"""
        if self.metadata is None:
            self.metadata = {}
        if self.dependencies is None:
            self.dependencies = []
        if self.subtasks is None:
            self.subtasks = []

    def __getitem__(self, key):
        return getattr(self, key)

    def __setitem__(self, key, value):
        setattr(self, key, value)
__post_init__()

Ensure all mutable defaults are properly initialized

Source code in toolboxv2/mods/isaa/base/Agent/types.py
449
450
451
452
453
454
455
456
def __post_init__(self):
    """Ensure all mutable defaults are properly initialized"""
    if self.metadata is None:
        self.metadata = {}
    if self.dependencies is None:
        self.dependencies = []
    if self.subtasks is None:
        self.subtasks = []
TaskExecutorNode

Bases: AsyncNode

Vollständige Task-Ausführung als unabhängige Node mit LLM-unterstützter Planung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
@with_progress_tracking
class TaskExecutorNode(AsyncNode):
    """Vollständige Task-Ausführung als unabhängige Node mit LLM-unterstützter Planung"""

    def __init__(self, max_parallel: int = 3, **kwargs):
        super().__init__(**kwargs)
        self.max_parallel = max_parallel
        self.results_store = {}  # Für {{ }} Referenzen
        self.execution_history = []  # Für LLM-basierte Optimierung
        self.agent_instance = None  # Wird gesetzt vom FlowAgent
        self.variable_manager = None
        self.fast_llm_model = None
        self.complex_llm_model = None
        self.progress_tracker = None

    async def prep_async(self, shared):
        """Enhanced preparation with unified variable system"""
        current_plan = shared.get("current_plan")
        tasks = shared.get("tasks", {})

        # Get unified variable manager
        self.variable_manager = shared.get("variable_manager")
        self.progress_tracker = shared.get("progress_tracker")
        if not self.variable_manager:
            self.variable_manager = VariableManager(shared.get("world_model", {}), shared)

        # Register all necessary scopes
        self.variable_manager.set_results_store(self.results_store)
        self.variable_manager.set_tasks_store(tasks)
        self.variable_manager.register_scope('user', shared.get('user_context', {}))
        self.variable_manager.register_scope('system', {
            'timestamp': datetime.now().isoformat(),
            'agent_name': shared.get('agent_instance', {}).amd.name if shared.get('agent_instance') else 'unknown'
        })

        # Stelle sicher, dass Agent-Referenz verfügbar ist
        if not self.agent_instance:
            self.agent_instance = shared.get("agent_instance")

        if not current_plan:
            return {"error": "No active plan", "tasks": tasks}

        # Rest of existing prep_async logic...
        ready_tasks = self._find_ready_tasks(current_plan, tasks)
        blocked_tasks = self._find_blocked_tasks(current_plan, tasks)

        execution_plan = await self._create_intelligent_execution_plan(
            ready_tasks, blocked_tasks, current_plan, shared
        )
        self.complex_llm_model = shared.get("complex_llm_model")
        self.fast_llm_model = shared.get("fast_llm_model")

        return {
            "plan": current_plan,
            "ready_tasks": ready_tasks,
            "blocked_tasks": blocked_tasks,
            "all_tasks": tasks,
            "execution_plan": execution_plan,
            "fast_llm_model": self.fast_llm_model,
            "complex_llm_model": self.complex_llm_model,
            "available_tools": shared.get("available_tools", []),
            "world_model": shared.get("world_model", {}),
            "results": self.results_store,
            "variable_manager": self.variable_manager,
            "progress_tracker": self.progress_tracker ,
        }

    def _find_ready_tasks(self, plan: TaskPlan, all_tasks: dict[str, Task]) -> list[Task]:
        """Finde Tasks die zur Ausführung bereit sind"""
        ready = []
        for task in plan.tasks:
            if task.status == "pending" and self._dependencies_satisfied(task, all_tasks):
                ready.append(task)
        return ready

    def _find_blocked_tasks(self, plan: TaskPlan, all_tasks: dict[str, Task]) -> list[Task]:
        """Finde blockierte Tasks für Analyse"""
        blocked = []
        for task in plan.tasks:
            if task.status == "pending" and not self._dependencies_satisfied(task, all_tasks):
                blocked.append(task)
        return blocked

    def _dependencies_satisfied(self, task: Task, all_tasks: dict[str, Task]) -> bool:
        """Prüfe ob alle Dependencies erfüllt sind"""
        for dep_id in task.dependencies:
            if dep_id in all_tasks:
                dep_task = all_tasks[dep_id]
                if dep_task.status not in ["completed"]:
                    return False
            else:
                # Dependency existiert nicht - könnte Problem sein
                wprint(f"Task {task.id} has missing dependency: {dep_id}")
                return False
        return True

    async def _create_intelligent_execution_plan(
        self,
        ready_tasks: list[Task],
        blocked_tasks: list[Task],
        plan: TaskPlan,
        shared: dict
    ) -> dict[str, Any]:
        """LLM-unterstützte intelligente Ausführungsplanung"""

        if not ready_tasks:
            return {
                "strategy": "waiting",
                "reason": "No ready tasks",
                "blocked_count": len(blocked_tasks),
                "recommendations": []
            }

        # Einfache Planung für wenige Tasks
        if len(ready_tasks) <= 2 and not LITELLM_AVAILABLE:
            return self._create_simple_execution_plan(ready_tasks, plan)

        # LLM-basierte intelligente Planung
        return await self._llm_execution_planning(ready_tasks, blocked_tasks, plan, shared)

    def _create_simple_execution_plan(self, ready_tasks: list[Task], plan: TaskPlan) -> dict[str, Any]:
        """Einfache heuristische Ausführungsplanung"""

        # Prioritäts-basierte Sortierung
        sorted_tasks = sorted(ready_tasks, key=lambda t: (t.priority, t.created_at))

        # Parallelisierbare Tasks identifizieren
        parallel_groups = []
        current_group = []

        for task in sorted_tasks:
            # ToolTasks können oft parallel laufen
            if isinstance(task, ToolTask) and len(current_group) < self.max_parallel:
                current_group.append(task)
            else:
                if current_group:
                    parallel_groups.append(current_group)
                    current_group = []
                current_group.append(task)

        if current_group:
            parallel_groups.append(current_group)

        strategy = "parallel" if len(parallel_groups) > 1 or len(parallel_groups[0]) > 1 else "sequential"

        return {
            "strategy": strategy,
            "execution_groups": parallel_groups,
            "total_groups": len(parallel_groups),
            "reasoning": "Simple heuristic: priority-based with tool parallelization",
            "estimated_duration": self._estimate_duration(sorted_tasks)
        }

    async def _llm_execution_planning(
        self,
        ready_tasks: list[Task],
        blocked_tasks: list[Task],
        plan: TaskPlan,
        shared: dict
    ) -> dict[str, Any]:
        """Erweiterte LLM-basierte Ausführungsplanung"""

        try:
            # Erstelle detaillierte Task-Analyse für LLM
            task_analysis = self._analyze_tasks_for_llm(ready_tasks, blocked_tasks)
            execution_context = self._build_execution_context(shared)

            prompt = f"""
Du bist ein Experte für Task-Ausführungsplanung. Analysiere die verfügbaren Tasks und erstelle einen optimalen Ausführungsplan.

## Verfügbare Tasks zur Ausführung
{task_analysis['ready_tasks_summary']}

## Blockierte Tasks (zur Information)
{task_analysis['blocked_tasks_summary']}

## Ausführungskontext
- Max parallele Tasks: {self.max_parallel}
- Plan-Strategie: {plan.execution_strategy}
- Verfügbare Tools: {', '.join(shared.get('available_tools', []))}
- Bisherige Ergebnisse: {len(self.results_store)} Tasks abgeschlossen
- Execution History: {len(self.execution_history)} vorherige Zyklen

## Bisherige Performance
{execution_context}

## Aufgabe
Erstelle einen optimierten Ausführungsplan. Berücksichtige:
1. Task-Abhängigkeiten und Prioritäten
2. Parallelisierungsmöglichkeiten
3. Resource-Optimierung (Tools, LLM-Aufrufe)
4. Fehlerwahrscheinlichkeit und Retry-Strategien
5. Dynamische Argument-Auflösung zwischen Tasks

Antworte mit YAML:

```yaml
strategy: "parallel"  # "parallel" | "sequential" | "hybrid"
execution_groups:
  - group_id: 1
    tasks: ["task_1", "task_2"]  # Task IDs
    execution_mode: "parallel"
    priority: "high"
    estimated_duration: 30  # seconds
    risk_level: "low"  # low | medium | high
    dependencies_resolved: true
  - group_id: 2
    tasks: ["task_3"]
    execution_mode: "sequential"
    priority: "medium"
    estimated_duration: 15
    depends_on_groups: [1]
reasoning: "Detailed explanation of the execution strategy"
optimization_suggestions:
  - "Specific optimization 1"
  - "Specific optimization 2"
risk_mitigation:
  - risk: "Tool timeout"
    mitigation: "Use shorter timeout for parallel calls"
  - risk: "Argument resolution failure"
    mitigation: "Validate references before execution"
total_estimated_duration: 45
confidence: 0.85
```"""

            model_to_use = shared.get("complex_llm_model", "openrouter/openai/gpt-4o")

            content = await self.agent_instance.a_run_llm_completion(
                model=model_to_use,
                messages=[{"role": "user", "content": prompt}],
                temperature=0.3,
                max_tokens=2000,
                node_name="TaskExecutorNode", task_id="llm_execution_planning"
            )

            yaml_match = re.search(r"```yaml\s*(.*?)\s*```", content, re.DOTALL)
            yaml_str = yaml_match.group(1) if yaml_match else content.strip()

            execution_plan = yaml.safe_load(yaml_str)

            # Validiere und erweitere den Plan
            validated_plan = self._validate_execution_plan(execution_plan, ready_tasks)

            rprint(
                f"LLM execution plan created: {validated_plan.get('strategy')} with {len(validated_plan.get('execution_groups', []))} groups")
            return validated_plan

        except Exception as e:
            eprint(f"LLM execution planning failed: {e}")
            return self._create_simple_execution_plan(ready_tasks, plan)

    def _analyze_tasks_for_llm(self, ready_tasks: list[Task], blocked_tasks: list[Task]) -> dict[str, str]:
        """Analysiere Tasks für LLM-Prompt"""

        ready_summary = []
        for task in ready_tasks:
            task_info = f"- {task.id} ({task.type}): {task.description}"
            if hasattr(task, 'priority'):
                task_info += f" [Priority: {task.priority}]"
            if isinstance(task, ToolTask):
                task_info += f" [Tool: {task.tool_name}]"
                if task.arguments:
                    # Zeige dynamische Referenzen
                    dynamic_refs = [arg for arg in task.arguments.values() if isinstance(arg, str) and "{{" in arg]
                    if dynamic_refs:
                        task_info += f" [Dynamic refs: {len(dynamic_refs)}]"
            ready_summary.append(task_info)

        blocked_summary = []
        for task in blocked_tasks:
            deps = ", ".join(task.dependencies) if task.dependencies else "None"
            blocked_summary.append(f"- {task.id}: waiting for [{deps}]")

        return {
            "ready_tasks_summary": "\n".join(ready_summary) or "No ready tasks",
            "blocked_tasks_summary": "\n".join(blocked_summary) or "No blocked tasks"
        }

    def _build_execution_context(self, shared: dict) -> str:
        """Baue Kontext für LLM-Planung"""
        context_parts = []

        # Performance der letzten Executions
        if self.execution_history:
            recent = self.execution_history[-3:]  # Last 3 executions
            avg_duration = sum(h.get("duration", 0) for h in recent) / len(recent)
            success_rate = sum(1 for h in recent if h.get("success", False)) / len(recent)
            context_parts.append(f"Recent performance: {avg_duration:.1f}s avg, {success_rate:.1%} success rate")

        # Resource utilization
        if self.results_store:
            tool_usage = {}
            for task_result in self.results_store.values():
                metadata = task_result.get("metadata", {})
                task_type = metadata.get("task_type", "unknown")
                tool_usage[task_type] = tool_usage.get(task_type, 0) + 1
            context_parts.append(f"Resource usage: {tool_usage}")

        return "\n".join(context_parts) if context_parts else "No previous execution history"

    def _validate_execution_plan(self, plan: dict, ready_tasks: list[Task]) -> dict:
        """Validiere und korrigiere LLM-generierten Ausführungsplan"""

        # Standard-Werte setzen
        validated = {
            "strategy": plan.get("strategy", "sequential"),
            "execution_groups": [],
            "reasoning": plan.get("reasoning", "LLM-generated plan"),
            "total_estimated_duration": plan.get("total_estimated_duration", 60),
            "confidence": min(1.0, max(0.0, plan.get("confidence", 0.5)))
        }

        # Validiere execution groups
        task_ids_available = [t.id for t in ready_tasks]

        for group_data in plan.get("execution_groups", []):
            group_tasks = group_data.get("tasks", [])
            # Filtere nur verfügbare Tasks
            valid_tasks = [tid for tid in group_tasks if tid in task_ids_available]

            if valid_tasks:
                validated["execution_groups"].append({
                    "group_id": group_data.get("group_id", len(validated["execution_groups"]) + 1),
                    "tasks": valid_tasks,
                    "execution_mode": group_data.get("execution_mode", "sequential"),
                    "priority": group_data.get("priority", "medium"),
                    "estimated_duration": group_data.get("estimated_duration", 30),
                    "risk_level": group_data.get("risk_level", "medium")
                })

        # Falls keine validen Groups, erstelle Fallback
        if not validated["execution_groups"]:
            validated["execution_groups"] = [{
                "group_id": 1,
                "tasks": task_ids_available[:self.max_parallel],
                "execution_mode": "parallel",
                "priority": "high"
            }]

        return validated

    def _estimate_duration(self, tasks: list[Task]) -> int:
        """Schätze Ausführungsdauer in Sekunden"""
        duration = 0
        for task in tasks:
            if isinstance(task, ToolTask):
                duration += 10  # Tool calls meist schneller
            elif isinstance(task, LLMTask):
                duration += 20  # LLM calls brauchen länger
            else:
                duration += 15  # Standard
        return duration

    async def exec_async(self, prep_res):
        """Hauptausführungslogik mit intelligentem Routing"""

        if "error" in prep_res:
            return {"error": prep_res["error"]}

        execution_plan = prep_res["execution_plan"]

        if execution_plan["strategy"] == "waiting":
            return {
                "status": "waiting",
                "message": execution_plan["reason"],
                "blocked_count": execution_plan.get("blocked_count", 0)
            }

        # Starte Ausführung basierend auf Plan
        execution_start = datetime.now()

        try:
            if execution_plan["strategy"] == "parallel":
                results = await self._execute_parallel_plan(execution_plan, prep_res)
            elif execution_plan["strategy"] == "sequential":
                results = await self._execute_sequential_plan(execution_plan, prep_res)
            else:  # hybrid
                results = await self._execute_hybrid_plan(execution_plan, prep_res)

            execution_duration = (datetime.now() - execution_start).total_seconds()

            # Speichere Execution-History für LLM-Optimierung
            self.execution_history.append({
                "timestamp": execution_start.isoformat(),
                "strategy": execution_plan["strategy"],
                "duration": execution_duration,
                "tasks_executed": len(results),
                "success": all(r.get("status") == "completed" for r in results),
                "plan_confidence": execution_plan.get("confidence", 0.5)
            })

            # Behalte nur letzte 10 Executions
            if len(self.execution_history) > 10:
                self.execution_history = self.execution_history[-10:]

            return {
                "status": "executed",
                "results": results,
                "execution_duration": execution_duration,
                "strategy_used": execution_plan["strategy"],
                "completed_tasks": len([r for r in results if r.get("status") == "completed"]),
                "failed_tasks": len([r for r in results if r.get("status") == "failed"])
            }

        except Exception as e:
            eprint(f"Execution plan failed: {e}")
            return {
                "status": "execution_failed",
                "error": str(e),
                "results": []
            }

    async def _execute_parallel_plan(self, plan: dict, prep_res: dict) -> list[dict]:
        """Führe Plan mit parallelen Gruppen aus"""
        all_results = []

        for group in plan["execution_groups"]:
            group_tasks = self._get_tasks_by_ids(group["tasks"], prep_res)

            if group.get("execution_mode") == "parallel":
                # Parallele Ausführung innerhalb der Gruppe
                batch_results = await self._execute_parallel_batch(group_tasks)
            else:
                # Sequenzielle Ausführung innerhalb der Gruppe
                batch_results = await self._execute_sequential_batch(group_tasks)

            all_results.extend(batch_results)

            # Prüfe ob kritische Tasks fehlgeschlagen sind
            critical_failures = [
                r for r in batch_results
                if r.get("status") == "failed" and self._is_critical_task(r.get("task_id"), prep_res)
            ]

            if critical_failures:
                eprint(f"Critical task failures in group {group['group_id']}, stopping execution")
                break

        return all_results

    async def _execute_sequential_plan(self, plan: dict, prep_res: dict) -> list[dict]:
        """Führe Plan sequenziell aus"""
        all_results = []

        for group in plan["execution_groups"]:
            group_tasks = self._get_tasks_by_ids(group["tasks"], prep_res)
            batch_results = await self._execute_sequential_batch(group_tasks)
            all_results.extend(batch_results)

            # Stoppe bei kritischen Fehlern
            critical_failures = [
                r for r in batch_results
                if r.get("status") == "failed" and self._is_critical_task(r.get("task_id"), prep_res)
            ]

            if critical_failures:
                break

        return all_results

    async def _execute_hybrid_plan(self, plan: dict, prep_res: dict) -> list[dict]:
        """Hybride Ausführung - Groups parallel, innerhalb je nach Mode"""

        # Führe Gruppen parallel aus (wenn möglich)
        group_tasks_list = []
        for group in plan["execution_groups"]:
            group_tasks = self._get_tasks_by_ids(group["tasks"], prep_res)
            group_tasks_list.append((group, group_tasks))

        # Führe bis zu max_parallel Gruppen parallel aus
        batch_size = min(len(group_tasks_list), self.max_parallel)
        all_results = []

        for i in range(0, len(group_tasks_list), batch_size):
            batch = group_tasks_list[i:i + batch_size]

            # Erstelle Coroutines für jede Gruppe
            group_coroutines = []
            for group, tasks in batch:
                if group.get("execution_mode") == "parallel":
                    coro = self._execute_parallel_batch(tasks)
                else:
                    coro = self._execute_sequential_batch(tasks)
                group_coroutines.append(coro)

            # Führe Gruppen-Batch parallel aus
            batch_results = await asyncio.gather(*group_coroutines, return_exceptions=True)

            # Flache Liste der Ergebnisse
            for result_group in batch_results:
                if isinstance(result_group, Exception):
                    eprint(f"Group execution failed: {result_group}")
                    continue
                all_results.extend(result_group)

        return all_results

    def _get_tasks_by_ids(self, task_ids: list[str], prep_res: dict) -> list[Task]:
        """Hole Task-Objekte basierend auf IDs"""
        all_tasks = prep_res["all_tasks"]
        return [all_tasks[tid] for tid in task_ids if tid in all_tasks]

    def _is_critical_task(self, task_id: str, prep_res: dict) -> bool:
        """Prüfe ob Task kritisch ist"""
        task = prep_res["all_tasks"].get(task_id)
        if not task:
            return False
        return getattr(task, 'critical', False) or task.priority == 1

    async def _execute_parallel_batch(self, tasks: list[Task]) -> list[dict]:
        """Führe Tasks parallel aus"""
        if not tasks:
            return []

        # Limitiere auf max_parallel
        batch_size = min(len(tasks), self.max_parallel)
        batches = [tasks[i:i + batch_size] for i in range(0, len(tasks), batch_size)]

        all_results = []
        for batch in batches:
            batch_results = await asyncio.gather(
                *[self._execute_single_task(task) for task in batch],
                return_exceptions=True
            )

            # Handle exceptions
            processed_results = []
            for i, result in enumerate(batch_results):
                if isinstance(result, Exception):
                    eprint(f"Task {batch[i].id} failed with exception: {result}")
                    processed_results.append({
                        "task_id": batch[i].id,
                        "status": "failed",
                        "error": str(result)
                    })
                else:
                    processed_results.append(result)

            all_results.extend(processed_results)

        return all_results

    async def _execute_sequential_batch(self, tasks: list[Task]) -> list[dict]:
        """Führe Tasks sequenziell aus"""
        results = []

        for task in tasks:
            try:
                result = await self._execute_single_task(task)
                results.append(result)

                # Stoppe bei kritischen Fehlern in sequenzieller Ausführung
                if result.get("status") == "failed" and getattr(task, 'critical', False):
                    eprint(f"Critical task {task.id} failed, stopping sequential execution")
                    break

            except Exception as e:
                eprint(f"Sequential task {task.id} failed: {e}")
                results.append({
                    "task_id": task.id,
                    "status": "failed",
                    "error": str(e)
                })

                if getattr(task, 'critical', False):
                    break

        return results

    async def _execute_single_task(self, task: Task) -> dict:
        """Enhanced task execution with unified LLMToolNode usage"""
        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="task_start",
                node_name="TaskExecutorNode",
                status=NodeStatus.RUNNING,
                task_id=task.id,
                plan_id=self.variable_manager.get("shared.current_plan.id"),
                metadata={
                    "description": task.description,
                    "type": task.type,
                    "priority": task.priority,
                    "dependencies": task.dependencies
                }
            ))

        task_start = time.perf_counter()
        try:
            task.status = "running"
            task.started_at = datetime.now()

            # Ensure metadata is initialized
            if not hasattr(task, 'metadata') or task.metadata is None:
                task.metadata = {}

            # Pre-process task with variable resolution
            if isinstance(task, ToolTask):
                resolved_args = self._resolve_task_variables(task.arguments)
                result = await self._execute_tool_task_with_validation(task, resolved_args)
            elif isinstance(task, LLMTask):
                # Use LLMToolNode for LLM tasks instead of direct execution
                result = await self._execute_llm_via_llmtool(task)
            elif isinstance(task, DecisionTask):
                # Enhanced decision task with context awareness
                result = await self._execute_decision_task_enhanced(task)
            else:
                # Use LLMToolNode for generic tasks as well
                result = await self._execute_generic_via_llmtool(task)

            # Store result in unified system
            self._store_task_result(task.id, result, True)

            task.result = result
            task.status = "completed"
            task.completed_at = datetime.now()

            task_duration = time.perf_counter() - task_start

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="task_complete",
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    plan_id=self.variable_manager.get("shared.current_plan.id"),
                    status=NodeStatus.COMPLETED,
                    success=True,
                    duration=task_duration,
                    metadata={
                        "result_type": type(result).__name__,
                        "description": task.description
                    }
                ))

            return {
                "task_id": task.id,
                "status": "completed",
                "result": result
            }

        except Exception as e:
            task.error = str(e)
            task.status = "failed"
            task.retry_count += 1

            # Store error in unified system
            self._store_task_result(task.id, None, False, str(e))
            task_duration = time.perf_counter() - task_start

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="task_error",  # Klarer Event-Typ
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    plan_id=self.variable_manager.get("shared.current_plan.id"),
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=task_duration,
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    },
                    metadata={
                        "retry_count": task.retry_count,
                        "description": task.description
                    }
                ))

            eprint(f"Task {task.id} failed: {e}")
            return {
                "task_id": task.id,
                "status": "failed",
                "error": str(e),
                "retry_count": task.retry_count
            }

    async def _resolve_dynamic_arguments(self, arguments: dict[str, Any]) -> dict[str, Any]:
        """Enhanced dynamic argument resolution with full variable system"""
        resolved = {}

        for key, value in arguments.items():
            if isinstance(value, str):
                # FIXED: Use unified variable manager for all resolution
                resolved_value = self.variable_manager.format_text(value)

                # Log if variables weren't resolved (debugging)
                if "{{" in resolved_value and "}}" in resolved_value:
                    wprint(f"Unresolved variables in argument '{key}': {resolved_value}")

                resolved[key] = resolved_value
            else:
                resolved[key] = value

        return resolved

    async def _execute_tool_task_with_validation(self, task: ToolTask, resolved_args: dict[str, Any]) -> Any:
        """Tool execution with improved error detection and validation"""

        if not task.tool_name:
            raise ValueError(f"ToolTask {task.id} missing tool_name")

        agent = self.agent_instance
        if not agent:
            raise ValueError("Agent instance not available for tool execution")

        tool_start = time.perf_counter()

        # Track tool call start
        if self.progress_tracker:
            await self.progress_tracker.emit_event(ProgressEvent(
                event_type="tool_call",
                timestamp=time.time(),
                node_name="TaskExecutorNode",
                status=NodeStatus.RUNNING,
                task_id=task.id,
                tool_name=task.tool_name,
                tool_args=resolved_args,
                metadata={
                    "task_type": "ToolTask",
                    "hypothesis": task.hypothesis,
                    "validation_criteria": task.validation_criteria
                }
            ))

        try:
            rprint(f"Executing tool {task.tool_name} with resolved args: {resolved_args}")

            # Execute tool with timeout and retry logic
            result = await self._execute_tool_with_retries(task.tool_name, resolved_args, agent)

            tool_duration = time.perf_counter() - tool_start

            # Validate result before marking as success
            is_valid_result = self._validate_tool_result(result, task)

            if not is_valid_result:
                raise ValueError(f"Tool {task.tool_name} returned invalid result: {type(result).__name__}")

            # Track successful tool call
            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",
                    timestamp=time.time(),
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    status=NodeStatus.COMPLETED,
                    tool_name=task.tool_name,
                    tool_args=resolved_args,
                    tool_result=result,
                    duration=tool_duration,
                    success=True,
                    metadata={
                        "task_type": "ToolTask",
                        "result_type": type(result).__name__,
                        "result_length": len(str(result)),
                        "validation_passed": is_valid_result
                    }
                ))

            # FIXED: Store in variable manager with correct path structure
            if self.variable_manager:
                self.variable_manager.set(f"results.{task.id}.data", result)
                self.variable_manager.set(f"tasks.{task.id}.result", result)

            return result

        except Exception as e:
            tool_duration = time.perf_counter() - tool_start
            import traceback
            print(traceback.format_exc())

            # Detailed error tracking
            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="tool_call",
                    timestamp=time.time(),
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    status=NodeStatus.FAILED,
                    tool_name=task.tool_name,
                    tool_args=resolved_args,
                    duration=tool_duration,
                    success=False,
                    tool_error=str(e),
                    metadata={
                        "task_type": "ToolTask",
                        "error_type": type(e).__name__,
                        "retry_attempted": hasattr(self, '_retry_count')
                    }
                ))

            eprint(f"Tool execution failed for {task.tool_name}: {e}")
            raise
    async def _execute_llm_via_llmtool(self, task: LLMTask) -> Any:
        """Execute LLM task via LLMToolNode for consistency"""

        # Prepare context for LLMToolNode
        llm_shared = {
            "current_task_description": task.description,
            "formatted_context": {
                "recent_interaction": f"Executing LLM task: {task.description}",
                "session_summary": "",
                "task_context": f"Task ID: {task.id}, Priority: {task.priority}"
            },
            "variable_manager": self.variable_manager,
            "agent_instance": self.agent_instance,
            "available_tools": self.agent_instance.shared.get("available_tools", []) if self.agent_instance else [],
            "tool_capabilities": self.agent_instance._tool_capabilities if self.agent_instance else {},
            "fast_llm_model": self.fast_llm_model,
            "complex_llm_model": self.complex_llm_model,
            "progress_tracker": self.progress_tracker,
            "session_id": getattr(self, 'session_id', 'task_executor'),
            "use_fast_response": task.llm_config.get("model_preference", "fast") == "fast"
        }

        # Create LLMToolNode instance
        llm_node = LLMToolNode()

        # Execute via LLMToolNode
        try:
            result = await llm_node.run_async(llm_shared)
            # shared["current_response"]
            # shared["tool_calls_made"]
            # shared["llm_tool_conversation"]
            # shared["synthesized_response"]
            return llm_shared["current_response"]
        except Exception as e:
            eprint(f"LLMToolNode execution failed for task {task.id}: {e}")
            # Fallback to direct execution
            import traceback
            print(traceback.format_exc())
            return await self._execute_llm_task_enhanced(task)

    async def _execute_llm_task_enhanced(self, task: LLMTask) -> Any:
        """Enhanced LLM task execution with unified variable system"""
        if not LITELLM_AVAILABLE:
            raise Exception("LiteLLM not available for LLM tasks")

        # Get model preference with variable support
        llm_config = task.llm_config
        model_preference = llm_config.get("model_preference", "fast")

        if model_preference == "complex":
            model_to_use = self.variable_manager.get("system.complex_llm_model", "openrouter/openai/gpt-4o")
        else:
            model_to_use = self.variable_manager.get("system.fast_llm_model", "openrouter/anthropic/claude-3-haiku")

        # Build context for prompt
        context_data = {}
        for context_key in task.context_keys:
            value = self.variable_manager.get(context_key)
            if value is not None:
                context_data[context_key] = value

        # Resolve prompt template with full variable system
        final_prompt = self.variable_manager.format_text(
            task.prompt_template,
            context=context_data
        )

        llm_start = time.perf_counter()

        try:

            response = await litellm.acompletion(
                model=model_to_use,
                messages=[{"role": "user", "content": final_prompt}],
                temperature=llm_config.get("temperature", 0.7),
                max_tokens=llm_config.get("max_tokens", 2048)
            )

            result = response


            # Store intermediate result for other tasks
            self.variable_manager.set(f"tasks.{task.id}.result", result)

            # Output schema validation if present
            if task.output_schema:
                stripped = result.strip()

                try:
                    # Try JSON first if it looks like JSON
                    if stripped.startswith('{') or stripped.startswith('['):
                        parsed = json.loads(stripped)
                    else:
                        parsed = yaml.safe_load(stripped)

                    # Ensure metadata is a dict before updating
                    if not isinstance(task.metadata, dict):
                        task.metadata = {}

                    # Save parsed result
                    task.metadata["parsed_output"] = parsed

                except (json.JSONDecodeError, yaml.YAMLError):
                    # Save info about failure without logging output
                    if not isinstance(task.metadata, dict):
                        task.metadata = {}
                    task.metadata["parsed_output_error"] = "Invalid JSON/YAML format"

                except Exception as e:
                    if not isinstance(task.metadata, dict):
                        task.metadata = {}
                    task.metadata["parsed_output_error"] = f"Unexpected error: {str(e)}"

            return result
        except Exception as e:
            llm_duration = time.perf_counter() - llm_start

            if self.progress_tracker:
                await self.progress_tracker.emit_event(ProgressEvent(
                    event_type="llm_call",
                    node_name="TaskExecutorNode",
                    task_id=task.id,
                    status=NodeStatus.FAILED,
                    success=False,
                    duration=llm_duration,
                    llm_model=model_to_use,
                    error_details={
                        "message": str(e),
                        "type": type(e).__name__
                    }
                ))

            raise

    async def _execute_generic_via_llmtool(self, task: Task) -> Any:
        """
        Execute a generic task by treating its description as a query for the LLMToolNode.
        This provides a flexible fallback for undefined task types, leveraging the full
        reasoning and tool-use capabilities of the LLMToolNode.
        """
        # Prepare a shared context dictionary for the LLMToolNode, treating the
        # generic task's description as the primary query.
        llm_shared = {
            "current_task_description": task.description,
            "current_query": task.description,
            "formatted_context": {
                "recent_interaction": f"Executing generic task: {task.description}",
                "session_summary": f"The system needs to complete the following task: {task.description}",
                "task_context": f"Task ID: {task.id}, Priority: {task.priority}, Type: Generic"
            },
            "variable_manager": self.variable_manager,
            "agent_instance": self.agent_instance,
            # Generic tasks might require tools, so provide full tool context.
            "available_tools": self.agent_instance.shared.get("available_tools", []) if self.agent_instance else [],
            "tool_capabilities": self.agent_instance._tool_capabilities if self.agent_instance else {},
            "fast_llm_model": self.fast_llm_model,
            "complex_llm_model": self.complex_llm_model,
            "progress_tracker": self.progress_tracker,
            "session_id": getattr(self, 'session_id', 'task_executor_generic'),
            # Default to a fast model, assuming generic tasks are often straightforward.
            "use_fast_response": True
        }

        # Instantiate the LLMToolNode for this specific execution.
        llm_node = LLMToolNode()

        try:
            # Execute the node. It will run its internal loop for reasoning, tool calling, and response generation.
            # The results of the execution will be populated back into the `llm_shared` dictionary.
            await llm_node.run_async(llm_shared)

            # Extract the final response from the shared context populated by the node.
            # Prioritize the structured 'synthesized_response' but fall back to 'current_response'.
            final_response = llm_shared.get("synthesized_response", {}).get("synthesized_response")
            if not final_response:
                final_response = llm_shared.get("current_response", f"Generic task '{task.id}' processed.")

            return final_response

        except Exception as e:
            eprint(f"LLMToolNode execution for generic task {task.id} failed: {e}")
            # Re-raise the exception to allow the higher-level execution loop in
            # _execute_single_task to catch and handle it appropriately (e.g., for retries).
            raise

    async def _execute_decision_task_enhanced(self, task: DecisionTask) -> str:
        """Enhanced DecisionTask with intelligent replan assessment"""

        if not LITELLM_AVAILABLE:
            raise Exception("LiteLLM not available for decision tasks")

        # Build comprehensive context for decision
        decision_context = self._build_decision_context(task)

        # Enhanced decision prompt with full context
        enhanced_prompt = f"""
You are making a critical routing decision for task execution. Analyze all context carefully.

## Current Situation
{task.decision_prompt}

## Execution Context
{decision_context}

## Available Routing Options
{json.dumps(task.routing_map, indent=2)}

## Decision Guidelines
1. Only trigger "replan_from_here" if there's a genuine failure that cannot be recovered
2. Use "route_to_task" for normal flow continuation
3. Consider the full context, not just immediate results
4. Be conservative with replanning - it's expensive and can cause loops

Based on ALL the context above, what is your decision?
Respond with EXACTLY one of these options: {', '.join(task.routing_map.keys())}

Your decision:"""

        model_to_use = self.fast_llm_model if hasattr(self, 'fast_llm_model') else "openrouter/anthropic/claude-3-haiku"

        try:
            response = await litellm.acompletion(
                model=model_to_use,
                messages=[{"role": "user", "content": enhanced_prompt}],
                temperature=0.1,
                max_tokens=50
            )

            decision = response.choices[0].message.content.strip().lower().split('\n')[0]

            # Find matching key (case-insensitive)
            matched_key = None
            for key in task.routing_map:
                if key.lower() == decision:
                    matched_key = key
                    break

            if not matched_key:
                wprint(f"Decision '{decision}' not in routing map, using first option")
                matched_key = list(task.routing_map.keys())[0] if task.routing_map else "continue"

            routing_instruction = task.routing_map.get(matched_key, matched_key)

            # Enhanced metadata with decision reasoning
            if not hasattr(task, 'metadata'):
                task.metadata = {}

            task.metadata.update({
                "decision_made": matched_key,
                "routing_instruction": routing_instruction,
                "decision_context": decision_context,
                "replan_justified": self._assess_replan_necessity(matched_key, routing_instruction, decision_context)
            })

            # Handle dynamic planning instructions
            if isinstance(routing_instruction, dict) and "action" in routing_instruction:
                action = routing_instruction["action"]

                if action == "replan_from_here":
                    # Add extensive context for replanning
                    task.metadata["replan_context"] = {
                        "new_goal": routing_instruction.get("new_goal", "Continue with alternative approach"),
                        "failure_reason": f"Decision task {task.id} determined: {matched_key}",
                        "original_task": task.id,
                        "context": routing_instruction.get("context", ""),
                        "execution_history": self._get_execution_history_summary(),
                        "failed_approaches": self._identify_failed_approaches(),
                        "success_indicators": self._identify_success_patterns()
                    }

                self.variable_manager.set(f"tasks.{task.id}.result", {
                    "decision": matched_key,
                    "action": action,
                    "instruction": routing_instruction,
                    "confidence": self._calculate_decision_confidence(decision_context)
                })

                return action

            else:
                # Traditional routing
                next_task_id = routing_instruction if isinstance(routing_instruction, str) else str(routing_instruction)

                task.metadata.update({
                    "next_task_id": next_task_id,
                    "routing_action": "route_to_task"
                })

                self.variable_manager.set(f"tasks.{task.id}.result", {
                    "decision": matched_key,
                    "next_task": next_task_id
                })

                return matched_key

        except Exception as e:
            eprint(f"Enhanced decision task failed: {e}")
            raise

    async def post_async(self, shared, prep_res, exec_res):
        """Erweiterte Post-Processing mit dynamischer Plan-Anpassung"""

        # Results store in shared state integrieren
        shared["results"] = self.results_store

        if exec_res is None or "error" in exec_res:
            shared["executor_performance"] = {"status": "error", "last_error": exec_res.get("error")}
            return "execution_error"

        if exec_res["status"] == "waiting":
            shared["executor_status"] = "waiting_for_dependencies"
            return "waiting"

        # Performance-Metriken speichern
        performance_data = {
            "execution_duration": exec_res.get("execution_duration", 0),
            "strategy_used": exec_res.get("strategy_used", "unknown"),
            "completed_tasks": exec_res.get("completed_tasks", 0),
            "failed_tasks": exec_res.get("failed_tasks", 0),
            "success_rate": exec_res.get("completed_tasks", 0) / max(len(exec_res.get("results", [])), 1),
            "timestamp": datetime.now().isoformat()
        }
        shared["executor_performance"] = performance_data

        # Check for dynamic planning actions
        planning_action_detected = False

        for result in exec_res.get("results", []):
            task_id = result["task_id"]
            if task_id in shared["tasks"]:
                task = shared["tasks"][task_id]
                task.status = result["status"]

                if result["status"] == "completed":
                    task.result = result["result"]

                    # Check for planning actions from DecisionTasks
                    if hasattr(task, 'metadata') and task.metadata:
                        routing_action = task.metadata.get("routing_action")

                        if routing_action == "replan_from_here":
                            shared["needs_dynamic_replan"] = True
                            shared["replan_context"] = task.metadata.get("replan_context", {})
                            planning_action_detected = True
                            rprint(f"Dynamic replan triggered by task {task_id}")

                        elif routing_action == "append_plan":
                            shared["needs_plan_append"] = True
                            shared["append_context"] = task.metadata.get("append_context", {})
                            planning_action_detected = True
                            rprint(f"Plan append triggered by task {task_id}")

                    # Store verification results if available
                    if result.get("verification"):
                        if not hasattr(task, 'metadata'):
                            task.metadata = {}
                        task.metadata["verification"] = result["verification"]

                elif result["status"] == "failed":
                    task.error = result.get("error", "Unknown error")

        # Return appropriate status based on planning actions
        if planning_action_detected:
            if shared.get("needs_dynamic_replan"):
                return "needs_dynamic_replan"  # Goes to PlanReflectorNode
            elif shared.get("needs_plan_append"):
                return "needs_plan_append"  # Goes to PlanReflectorNode

        # Regular completion checking
        current_plan = shared["current_plan"]
        if current_plan:
            all_finished = all(
                shared["tasks"][task.id].status in ["completed", "failed"]
                for task in current_plan.tasks
            )

            if all_finished:
                current_plan.status = "completed"
                shared["plan_completion_time"] = datetime.now().isoformat()
                rprint(f"Plan {current_plan.id} finished")
                return "plan_completed"
            else:
                ready_tasks = [
                    task for task in current_plan.tasks
                    if shared["tasks"][task.id].status == "pending"
                ]

                if ready_tasks:
                    return "continue_execution"
                else:
                    return "waiting"

        return "execution_complete"

    def get_execution_statistics(self) -> dict[str, Any]:
        """Erhalte detaillierte Ausführungsstatistiken"""
        if not self.execution_history:
            return {"message": "No execution history available"}

        history = self.execution_history

        return {
            "total_executions": len(history),
            "average_duration": sum(h["duration"] for h in history) / len(history),
            "success_rate": sum(1 for h in history if h["success"]) / len(history),
            "strategy_usage": {
                strategy: sum(1 for h in history if h["strategy"] == strategy)
                for strategy in set(h["strategy"] for h in history)
            },
            "total_tasks_executed": sum(h["tasks_executed"] for h in history),
            "average_confidence": sum(h["plan_confidence"] for h in history) / len(history),
            "recent_performance": history[-3:] if len(history) >= 3 else history
        }

    def _resolve_task_variables(self, data):
        """Unified variable resolution for any task data"""
        if isinstance(data, str):
            res = self.variable_manager.format_text(data)
            return res
        elif isinstance(data, dict):
            resolved = {}
            for key, value in data.items():
                resolved[key] = self._resolve_task_variables(value)
            return resolved
        elif isinstance(data, list):
            return [self._resolve_task_variables(item) for item in data]
        else:
            return data

    def _store_task_result(self, task_id: str, result: Any, success: bool, error: str = None):
        """Store task result in unified variable system"""
        result_data = {
            "data": result,
            "metadata": {
                "task_type": "task",
                "completed_at": datetime.now().isoformat(),
                "success": success
            }
        }

        if error:
            result_data["error"] = error
            result_data["metadata"]["success"] = False

        # Store in results_store and update variable manager
        self.results_store[task_id] = result_data
        self.variable_manager.set_results_store(self.results_store)

        # FIXED: Store actual result data, not the wrapper object
        self.variable_manager.set(f"results.{task_id}.data", result)
        self.variable_manager.set(f"results.{task_id}.metadata", result_data["metadata"])
        if error:
            self.variable_manager.set(f"results.{task_id}.error", error)

    def _build_decision_context(self, task: DecisionTask) -> str:
        """Build comprehensive context for decision making"""

        context_parts = []

        # Recent execution results
        recent_results = []
        for task_id, result_data in list(self.results_store.items())[-3:]:
            success = result_data.get("metadata", {}).get("success", False)
            status = "✓" if success else "✗"
            data_preview = str(result_data.get("data", ""))[:100] + "..."
            recent_results.append(f"{status} {task_id}: {data_preview}")

        if recent_results:
            context_parts.append("Recent Results:\n" + "\n".join(recent_results))

        # Variable context
        if self.variable_manager:
            available_vars = list(self.variable_manager.get_available_variables().keys())[:10]
            context_parts.append(f"Available Variables: {', '.join(available_vars)}")

        # Execution history
        execution_summary = self._get_execution_history_summary()
        if execution_summary:
            context_parts.append(f"Execution Summary: {execution_summary}")

        # Current world model insights
        world_insights = self._get_world_model_insights()
        if world_insights:
            context_parts.append(f"Known Facts: {world_insights}")

        return "\n\n".join(context_parts)

    def _assess_replan_necessity(self, decision: str, routing_instruction: Any, context: str) -> bool:
        """Assess if replanning is truly necessary"""

        if not isinstance(routing_instruction, dict):
            return False

        action = routing_instruction.get("action", "")
        if action != "replan_from_here":
            return False

        # Check if we have genuine failures
        genuine_failures = "error" in context.lower() or "failed" in context.lower()
        alternative_available = len(self.results_store) > 0  # Have some results to work with

        # Be conservative - only replan if really necessary
        return genuine_failures and not alternative_available

    async def _execute_tool_with_retries(self, tool_name: str, args: dict, agent, max_retries: int = 2) -> Any:
        """Execute tool with retry logic"""

        last_exception = None

        for attempt in range(max_retries + 1):
            try:
                result = await agent.arun_function(tool_name, **args)

                # Additional validation - check if result indicates success
                if self._is_tool_result_success(result):
                    return result
                elif attempt < max_retries:
                    wprint(f"Tool {tool_name} returned unclear result, retrying...")
                    continue
                else:
                    return result

            except Exception as e:
                last_exception = e
                if attempt < max_retries:
                    wprint(f"Tool {tool_name} failed (attempt {attempt + 1}), retrying: {e}")
                    # await asyncio.sleep(0.5 * (attempt + 1))  # Progressive delay
                else:
                    eprint(f"Tool {tool_name} failed after {max_retries + 1} attempts")

        if last_exception:
            raise last_exception
        else:
            raise RuntimeError(f"Tool {tool_name} failed without exception")

    def _validate_tool_result(self, result: Any, task: ToolTask) -> bool:
        """Validate tool result to prevent false failures"""

        # Basic validation
        if result is None:
            return False

        # Check for common error indicators
        if isinstance(result, str):
            error_indicators = ["error", "failed", "exception", "timeout", "not found"]
            result_lower = result.lower()

            # If result contains error indicators but also has substantial content, it might still be valid
            has_errors = any(indicator in result_lower for indicator in error_indicators)
            has_content = len(result.strip()) > 20

            if has_errors and not has_content:
                return False

        # Check against expectation if provided
        if hasattr(task, 'expectation') and task.expectation:
            expectation_keywords = task.expectation.lower().split()
            result_text = str(result).lower()

            # At least one expectation keyword should be present
            if not any(keyword in result_text for keyword in expectation_keywords):
                wprint(f"Tool result doesn't match expectation: {task.expectation}")

        return True

    def _is_tool_result_success(self, result: Any) -> bool:
        """Determine if a tool result indicates success"""

        if result is None:
            return False

        if isinstance(result, bool):
            return result

        if isinstance(result, list | dict):
            return len(result) > 0

        if isinstance(result, str):
            # Check for explicit success/failure indicators
            result_lower = result.lower()

            success_indicators = ["success", "completed", "found", "retrieved", "generated"]
            failure_indicators = ["error", "failed", "not found", "timeout", "exception"]

            has_success = any(indicator in result_lower for indicator in success_indicators)
            has_failure = any(indicator in result_lower for indicator in failure_indicators)

            if has_success and not has_failure:
                return True
            elif has_failure and not has_success:
                return False
            else:
                # Ambiguous - assume success if there's substantial content
                return len(result.strip()) > 10

        # For other types, assume success if not None
        return True

    def _get_execution_history_summary(self) -> str:
        """Get concise execution history summary"""

        if not hasattr(self, 'execution_history') or not self.execution_history:
            return "No execution history"

        recent = self.execution_history[-3:]  # Last 3 executions
        summaries = []

        for hist in recent:
            status = "Success" if hist.get("success", False) else "Failed"
            duration = hist.get("duration", 0)
            strategy = hist.get("strategy", "Unknown")
            summaries.append(f"{strategy}: {status} ({duration:.1f}s)")

        return "; ".join(summaries)

    def _identify_failed_approaches(self) -> list[str]:
        """Identify approaches that have consistently failed"""

        failed_approaches = []

        # Analyze failed tasks
        for _task_id, result_data in self.results_store.items():
            if not result_data.get("metadata", {}).get("success", True):
                error = result_data.get("error", "")
                if "tool" in error.lower():
                    failed_approaches.append("direct_tool_approach")
                elif "search" in error.lower():
                    failed_approaches.append("search_based_approach")
                elif "llm" in error.lower():
                    failed_approaches.append("llm_direct_approach")

        return list(set(failed_approaches))

    def _identify_success_patterns(self) -> list[str]:
        """Identify patterns that have led to success"""

        success_patterns = []

        # Analyze successful tasks
        successful_results = [
            r for r in self.results_store.values()
            if r.get("metadata", {}).get("success", False)
        ]

        if successful_results:
            # Identify common patterns
            if len(successful_results) > 1:
                success_patterns.append("multi_step_approach")

            for result in successful_results:
                data = result.get("data", "")
                if isinstance(data, str) and len(data) > 100:
                    success_patterns.append("detailed_information_retrieval")

        return list(set(success_patterns))

    def _get_world_model_insights(self) -> str:
        """Get relevant insights from world model"""

        if not self.variable_manager:
            return ""

        world_data = self.variable_manager.scopes.get("world", {})
        if not world_data:
            return "No world model data"

        # Get most recent or relevant facts
        recent_facts = []
        for key, value in list(world_data.items())[:5]:  # Top 5 facts
            recent_facts.append(f"{key}: {str(value)[:50]}...")

        return "; ".join(recent_facts)

    def _calculate_decision_confidence(self, context: str) -> float:
        """Calculate confidence in decision based on context"""

        # Simple heuristic based on context richness
        base_confidence = 0.5

        # Boost confidence if we have rich context
        if len(context) > 200:
            base_confidence += 0.2

        # Boost if we have recent results
        if "Recent Results:" in context:
            base_confidence += 0.2

        # Reduce if there are many failures
        failure_count = context.lower().count("failed") + context.lower().count("error")
        base_confidence -= min(failure_count * 0.1, 0.3)

        return max(0.1, min(1.0, base_confidence))
exec_async(prep_res) async

Hauptausführungslogik mit intelligentem Routing

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
async def exec_async(self, prep_res):
    """Hauptausführungslogik mit intelligentem Routing"""

    if "error" in prep_res:
        return {"error": prep_res["error"]}

    execution_plan = prep_res["execution_plan"]

    if execution_plan["strategy"] == "waiting":
        return {
            "status": "waiting",
            "message": execution_plan["reason"],
            "blocked_count": execution_plan.get("blocked_count", 0)
        }

    # Starte Ausführung basierend auf Plan
    execution_start = datetime.now()

    try:
        if execution_plan["strategy"] == "parallel":
            results = await self._execute_parallel_plan(execution_plan, prep_res)
        elif execution_plan["strategy"] == "sequential":
            results = await self._execute_sequential_plan(execution_plan, prep_res)
        else:  # hybrid
            results = await self._execute_hybrid_plan(execution_plan, prep_res)

        execution_duration = (datetime.now() - execution_start).total_seconds()

        # Speichere Execution-History für LLM-Optimierung
        self.execution_history.append({
            "timestamp": execution_start.isoformat(),
            "strategy": execution_plan["strategy"],
            "duration": execution_duration,
            "tasks_executed": len(results),
            "success": all(r.get("status") == "completed" for r in results),
            "plan_confidence": execution_plan.get("confidence", 0.5)
        })

        # Behalte nur letzte 10 Executions
        if len(self.execution_history) > 10:
            self.execution_history = self.execution_history[-10:]

        return {
            "status": "executed",
            "results": results,
            "execution_duration": execution_duration,
            "strategy_used": execution_plan["strategy"],
            "completed_tasks": len([r for r in results if r.get("status") == "completed"]),
            "failed_tasks": len([r for r in results if r.get("status") == "failed"])
        }

    except Exception as e:
        eprint(f"Execution plan failed: {e}")
        return {
            "status": "execution_failed",
            "error": str(e),
            "results": []
        }
get_execution_statistics()

Erhalte detaillierte Ausführungsstatistiken

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
def get_execution_statistics(self) -> dict[str, Any]:
    """Erhalte detaillierte Ausführungsstatistiken"""
    if not self.execution_history:
        return {"message": "No execution history available"}

    history = self.execution_history

    return {
        "total_executions": len(history),
        "average_duration": sum(h["duration"] for h in history) / len(history),
        "success_rate": sum(1 for h in history if h["success"]) / len(history),
        "strategy_usage": {
            strategy: sum(1 for h in history if h["strategy"] == strategy)
            for strategy in set(h["strategy"] for h in history)
        },
        "total_tasks_executed": sum(h["tasks_executed"] for h in history),
        "average_confidence": sum(h["plan_confidence"] for h in history) / len(history),
        "recent_performance": history[-3:] if len(history) >= 3 else history
    }
post_async(shared, prep_res, exec_res) async

Erweiterte Post-Processing mit dynamischer Plan-Anpassung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
async def post_async(self, shared, prep_res, exec_res):
    """Erweiterte Post-Processing mit dynamischer Plan-Anpassung"""

    # Results store in shared state integrieren
    shared["results"] = self.results_store

    if exec_res is None or "error" in exec_res:
        shared["executor_performance"] = {"status": "error", "last_error": exec_res.get("error")}
        return "execution_error"

    if exec_res["status"] == "waiting":
        shared["executor_status"] = "waiting_for_dependencies"
        return "waiting"

    # Performance-Metriken speichern
    performance_data = {
        "execution_duration": exec_res.get("execution_duration", 0),
        "strategy_used": exec_res.get("strategy_used", "unknown"),
        "completed_tasks": exec_res.get("completed_tasks", 0),
        "failed_tasks": exec_res.get("failed_tasks", 0),
        "success_rate": exec_res.get("completed_tasks", 0) / max(len(exec_res.get("results", [])), 1),
        "timestamp": datetime.now().isoformat()
    }
    shared["executor_performance"] = performance_data

    # Check for dynamic planning actions
    planning_action_detected = False

    for result in exec_res.get("results", []):
        task_id = result["task_id"]
        if task_id in shared["tasks"]:
            task = shared["tasks"][task_id]
            task.status = result["status"]

            if result["status"] == "completed":
                task.result = result["result"]

                # Check for planning actions from DecisionTasks
                if hasattr(task, 'metadata') and task.metadata:
                    routing_action = task.metadata.get("routing_action")

                    if routing_action == "replan_from_here":
                        shared["needs_dynamic_replan"] = True
                        shared["replan_context"] = task.metadata.get("replan_context", {})
                        planning_action_detected = True
                        rprint(f"Dynamic replan triggered by task {task_id}")

                    elif routing_action == "append_plan":
                        shared["needs_plan_append"] = True
                        shared["append_context"] = task.metadata.get("append_context", {})
                        planning_action_detected = True
                        rprint(f"Plan append triggered by task {task_id}")

                # Store verification results if available
                if result.get("verification"):
                    if not hasattr(task, 'metadata'):
                        task.metadata = {}
                    task.metadata["verification"] = result["verification"]

            elif result["status"] == "failed":
                task.error = result.get("error", "Unknown error")

    # Return appropriate status based on planning actions
    if planning_action_detected:
        if shared.get("needs_dynamic_replan"):
            return "needs_dynamic_replan"  # Goes to PlanReflectorNode
        elif shared.get("needs_plan_append"):
            return "needs_plan_append"  # Goes to PlanReflectorNode

    # Regular completion checking
    current_plan = shared["current_plan"]
    if current_plan:
        all_finished = all(
            shared["tasks"][task.id].status in ["completed", "failed"]
            for task in current_plan.tasks
        )

        if all_finished:
            current_plan.status = "completed"
            shared["plan_completion_time"] = datetime.now().isoformat()
            rprint(f"Plan {current_plan.id} finished")
            return "plan_completed"
        else:
            ready_tasks = [
                task for task in current_plan.tasks
                if shared["tasks"][task.id].status == "pending"
            ]

            if ready_tasks:
                return "continue_execution"
            else:
                return "waiting"

    return "execution_complete"
prep_async(shared) async

Enhanced preparation with unified variable system

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
async def prep_async(self, shared):
    """Enhanced preparation with unified variable system"""
    current_plan = shared.get("current_plan")
    tasks = shared.get("tasks", {})

    # Get unified variable manager
    self.variable_manager = shared.get("variable_manager")
    self.progress_tracker = shared.get("progress_tracker")
    if not self.variable_manager:
        self.variable_manager = VariableManager(shared.get("world_model", {}), shared)

    # Register all necessary scopes
    self.variable_manager.set_results_store(self.results_store)
    self.variable_manager.set_tasks_store(tasks)
    self.variable_manager.register_scope('user', shared.get('user_context', {}))
    self.variable_manager.register_scope('system', {
        'timestamp': datetime.now().isoformat(),
        'agent_name': shared.get('agent_instance', {}).amd.name if shared.get('agent_instance') else 'unknown'
    })

    # Stelle sicher, dass Agent-Referenz verfügbar ist
    if not self.agent_instance:
        self.agent_instance = shared.get("agent_instance")

    if not current_plan:
        return {"error": "No active plan", "tasks": tasks}

    # Rest of existing prep_async logic...
    ready_tasks = self._find_ready_tasks(current_plan, tasks)
    blocked_tasks = self._find_blocked_tasks(current_plan, tasks)

    execution_plan = await self._create_intelligent_execution_plan(
        ready_tasks, blocked_tasks, current_plan, shared
    )
    self.complex_llm_model = shared.get("complex_llm_model")
    self.fast_llm_model = shared.get("fast_llm_model")

    return {
        "plan": current_plan,
        "ready_tasks": ready_tasks,
        "blocked_tasks": blocked_tasks,
        "all_tasks": tasks,
        "execution_plan": execution_plan,
        "fast_llm_model": self.fast_llm_model,
        "complex_llm_model": self.complex_llm_model,
        "available_tools": shared.get("available_tools", []),
        "world_model": shared.get("world_model", {}),
        "results": self.results_store,
        "variable_manager": self.variable_manager,
        "progress_tracker": self.progress_tracker ,
    }
TaskManagementFlow

Bases: AsyncFlow

Enhanced Task-Management-Flow with LLMReasonerNode as strategic core. The flow now starts with strategic reasoning and delegates to specialized sub-systems.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
@with_progress_tracking
class TaskManagementFlow(AsyncFlow):
    """
    Enhanced Task-Management-Flow with LLMReasonerNode as strategic core.
    The flow now starts with strategic reasoning and delegates to specialized sub-systems.
    """

    def __init__(self, max_parallel_tasks: int = 3, max_reasoning_loops: int = 24, max_tool_calls:int = 5):
        # Create the strategic reasoning core (new primary node)
        self.llm_reasoner = LLMReasonerNode(max_reasoning_loops=max_reasoning_loops)

        # Create specialized sub-system nodes (now supporting nodes)
        self.planner_node = TaskPlannerNode()
        self.executor_node = TaskExecutorNode(max_parallel=max_parallel_tasks)
        self.sync_node = StateSyncNode()
        self.llm_tool_node = LLMToolNode(max_tool_calls=max_tool_calls)

        # Store references for the reasoner to access sub-systems
        # These will be injected into shared state during execution

        # === NEW HIERARCHICAL FLOW STRUCTURE ===

        # Primary flow: LLMReasonerNode is the main orchestrator
        # It makes strategic decisions and routes to appropriate sub-systems

        # The reasoner can internally call any of these sub-systems:
        # - LLMToolNode for direct tool usage
        # - TaskPlanner + TaskExecutor for complex project management
        # - Direct response for simple queries

        # Only one main connection: reasoner completes -> response generation
        self.llm_reasoner - "reasoner_complete" >> self.sync_node

        # Fallback connections for error handling
        self.llm_reasoner - "error" >> self.sync_node
        self.llm_reasoner - "timeout" >> self.sync_node

        # The old linear connections are removed - the reasoner now controls the flow internally

        super().__init__(start=self.llm_reasoner)

    async def run_async(self, shared):
        """Enhanced run with sub-system injection"""

        # Inject sub-system references into shared state so reasoner can access them
        shared["llm_tool_node_instance"] = self.llm_tool_node
        shared["task_planner_instance"] = self.planner_node
        shared["task_executor_instance"] = self.executor_node

        # Store tool registry access for the reasoner
        agent_instance = shared.get("agent_instance")
        if agent_instance:
            shared["tool_registry"] = agent_instance._tool_registry
            shared["tool_capabilities"] = agent_instance._tool_capabilities

        # Execute the flow with the reasoner as starting point
        return await super().run_async(shared)
run_async(shared) async

Enhanced run with sub-system injection

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
async def run_async(self, shared):
    """Enhanced run with sub-system injection"""

    # Inject sub-system references into shared state so reasoner can access them
    shared["llm_tool_node_instance"] = self.llm_tool_node
    shared["task_planner_instance"] = self.planner_node
    shared["task_executor_instance"] = self.executor_node

    # Store tool registry access for the reasoner
    agent_instance = shared.get("agent_instance")
    if agent_instance:
        shared["tool_registry"] = agent_instance._tool_registry
        shared["tool_capabilities"] = agent_instance._tool_capabilities

    # Execute the flow with the reasoner as starting point
    return await super().run_async(shared)
TaskPlannerNode

Bases: AsyncNode

Erweiterte Aufgabenplanung mit dynamischen Referenzen und Tool-Integration

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
@with_progress_tracking
class TaskPlannerNode(AsyncNode):
    """Erweiterte Aufgabenplanung mit dynamischen Referenzen und Tool-Integration"""

    async def prep_async(self, shared):
        """Enhanced preparation with goals-based planning support"""

        # Check if this is a goals-based call from LLMReasonerNode
        replan_context = shared.get("replan_context", {})
        goals_list = replan_context.get("goals", [])

        if goals_list:
            # Goals-based planning (called by LLMReasonerNode)
            return {
                "goals": goals_list,
                "planning_mode": "goals_based",
                "query": shared.get("current_query", ""),
                "reasoning_context": replan_context.get("reasoning_context", ""),
                "triggered_by": replan_context.get("triggered_by", "unknown"),
                "tasks": shared.get("tasks", {}),
                "system_status": shared.get("system_status", "idle"),
                "tool_capabilities": shared.get("tool_capabilities", {}),
                "available_tools_names": shared.get("available_tools", []),
                "strategy": "goals_decomposition",  # New strategy type
                "fast_llm_model": shared.get("fast_llm_model"),
                "complex_llm_model": shared.get("complex_llm_model"),
                "agent_instance": shared.get("agent_instance"),
                "variable_manager": shared.get("variable_manager"),
            }
        else:
            # Legacy planning (original query-based approach)
            return {
                "query": shared.get("current_query", ""),
                "tasks": shared.get("tasks", {}),
                "system_status": shared.get("system_status", "idle"),
                "tool_capabilities": shared.get("tool_capabilities", {}),
                "available_tools_names": shared.get("available_tools", []),
                "strategy": shared.get("selected_strategy", "direct_response"),
                "fast_llm_model": shared.get("fast_llm_model"),
                "complex_llm_model": shared.get("complex_llm_model"),
                "agent_instance": shared.get("agent_instance"),
                "variable_manager": shared.get("variable_manager"),
                "planning_mode": "legacy"
            }

    async def exec_async(self, prep_res):
        if prep_res["strategy"] == "fast_simple_planning":
            return await self._create_simple_plan(prep_res)
        else:
            return await self._advanced_llm_decomposition(prep_res)

    async def post_async(self, shared, prep_res, exec_res):
        """Post-processing nach Plan-Erstellung"""

        if exec_res is None:
            shared["planning_error"] = "Plan creation returned None"
            return "planning_failed"

        if isinstance(exec_res, TaskPlan):

            progress_tracker = shared.get("progress_tracker")
            if progress_tracker:
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="plan_created",
                    node_name="TaskPlannerNode",
                    session_id=shared.get("session_id"),
                    status=NodeStatus.COMPLETED,
                    success=True,
                    plan_id=exec_res.id,
                    metadata={
                        "plan_name": exec_res.name,
                        "task_count": len(exec_res.tasks),
                        "strategy": exec_res.execution_strategy
                    }
                ))

            # Erfolgreicher Plan
            shared["current_plan"] = exec_res

            # Tasks in shared state für Executor verfügbar machen
            task_dict = {task.id: task for task in exec_res.tasks}
            if "tasks" not in shared:
                shared["tasks"] = task_dict
            else:
                shared["tasks"].update(task_dict)

            # Plan-Metadaten setzen
            shared["plan_created_at"] = datetime.now().isoformat()
            shared["plan_strategy"] = exec_res.execution_strategy
            shared["total_tasks_planned"] = len(exec_res.tasks)

            rprint(f"Plan created successfully: {exec_res.name} with {len(exec_res.tasks)} tasks")
            return "planned"

        else:
            # Plan creation failed
            shared["planning_error"] = "Invalid plan format returned"
            shared["current_plan"] = None
            eprint("Plan creation failed - invalid format")
            return "planning_failed"

    async def _create_simple_plan(self, prep_res) -> TaskPlan:
        """Fast lightweight planning for direct or simple multi-step queries."""
        taw = self._build_tool_intelligence(prep_res)
        rprint("You are a FAST "+ taw)
        prompt = f"""
You are a FAST abstract pattern recognizer and task planner.
Identify if the query needs a **single-step LLM answer** or a **simple 2–3 task plan** using available tools.
Output ONLY YAML.

## User Query
{prep_res['query']}

## Available Tools
{taw}

## Pattern Recognition (Internal Only)
- Detect if query is informational, action-based, or tool-eligible.
- Map to minimal plan type: "direct_llm" or "simple_tool_plus_llm".

## YAML Schema
```yaml
plan_name: string
description: string
execution_strategy: "sequential" | "parallel"
tasks:
  - id: string
    type: "LLMTask" | "ToolTask"
    description: string
    priority: int
    dependencies: [list]
Example 1 — Direct LLM
```yaml
plan_name: Direct Response
description: Quick answer from LLM
execution_strategy: sequential
tasks:
  - id: answer
    type: LLMTask
    description: Respond to query
    priority: 1
    dependencies: []
    prompt_template: Respond concisely to: {prep_res['query']}
    llm_config:
      model_preference: fast
      temperature: 0.3
```
Example 2 — Tool + LLM
```yaml
plan_name: Fetch and Answer
description: Get info from tool and summarize
execution_strategy: sequential
tasks:
  - id: fetch_info
    type: ToolTask
    description: Get required data
    priority: 1
    dependencies: []
    tool_name: info_api
    arguments:
      query: "{prep_res['query']}"
  - id: summarize
    type: LLMTask
    description: Summarize fetched data
    priority: 2
    dependencies: ["fetch_info"]
    prompt_template: Summarize: {{ results.fetch_info.data }}
    llm_config:
      model_preference: fast
      temperature: 0.3
```
Output Requirements
Use ONLY YAML for the final output
Pick minimal plan type for fastest completion!
focus on correct quotation and correct yaml format!
    """

        try:
            agent_instance = prep_res["agent_instance"]
            plan_data = await agent_instance.a_format_class(PlanData,
                model=prep_res.get("complex_llm_model", "openrouter/anthropic/claude-3-haiku"),
                prompt= prompt,
                temperature=0.3,
                max_tokens=4512,
                auto_context=True,
                node_name="TaskPlannerNode",
                task_id="fast_simple_planning"
            )
            # print("Simple", json.dumps(plan_data, indent=2))
            return TaskPlan(
                id=str(uuid.uuid4()),
                name=plan_data.get("plan_name", "Generated Plan"),
                description=plan_data.get("description", f"Plan for: {prep_res['query']}"),
                tasks=[
                    [LLMTask, ToolTask, DecisionTask, Task][["LLMTask", "ToolTask", "DecisionTask", "Task"].index(t.get("type"))](**t)
                    for t in plan_data.get("tasks", [])
                ],
                execution_strategy=plan_data.get("execution_strategy", "sequential")
            )

        except Exception as e:
            eprint(f"Simple plan creation failed: {e}")
            import traceback
            print(traceback.format_exc())
            return TaskPlan(
                id=str(uuid.uuid4()),
                name="Fallback Plan",
                description="Direct response only",
                tasks=[
                    LLMTask(
                        id="fast_simple_planning",
                        type="LLMTask",
                        description="Generate direct response",
                        priority=1,
                        dependencies=[],
                        prompt_template=f"Respond to the query: {prep_res['query']}",
                        llm_config={"model_preference": "fast"}
                    )
                ]
            )

    async def _advanced_llm_decomposition(self, prep_res) -> TaskPlan:
        """Enhanced LLM-based decomposition with goals-based planning support"""

        planning_mode = prep_res.get("planning_mode", "legacy")
        variable_manager = prep_res.get("variable_manager")
        tool_intelligence = self._build_tool_intelligence(prep_res)

        if planning_mode == "goals_based":
            # Goals-based planning from LLMReasonerNode
            goals_list = prep_res.get("goals", [])
            reasoning_context = prep_res.get("reasoning_context", "")

            prompt = f"""
You are an expert task planner specialized in creating execution plans from strategic goals.
Create a comprehensive plan that addresses all goals with proper dependencies and parallelization.

## Strategic Goals from Reasoner
{chr(10).join([f"{i + 1}. {goal}" for i, goal in enumerate(goals_list)])}

## Reasoning Context
{reasoning_context}

## Your Available Tools & Intelligence
{tool_intelligence}

{variable_manager.get_llm_variable_context() if variable_manager else ""}

## Goals-Based Planning Instructions
1. Analyze each goal for dependencies on other goals
2. Identify goals that can be executed in parallel
3. Create tasks that address each goal effectively
4. Use variable references {{ results.task_id.data }} for dependencies
5. Ensure proper sequencing and coordination

## YAML Schema
```yaml
plan_name: string
description: string
execution_strategy: "sequential" | "parallel" | "mixed"
tasks:
  - id: string
    type: "LLMTask" | "ToolTask" | "DecisionTask"
    description: string
    priority: int
    dependencies: [list of task ids]
    # Type-specific fields as needed
Goals Decomposition Strategy

Independent Goals: Create parallel tasks
Sequential Goals: Use dependencies array
Complex Goals: Break into sub-tasks with DecisionTask routing
Data Dependencies: Use variable references between tasks

Example for Multi-Goal Plan
yamlCopyplan_name: "Multi-Goal Strategic Plan"
description: "Execute multiple strategic objectives with proper coordination"
execution_strategy: "mixed"
tasks:
  - id: "goal_1_research"
    type: "ToolTask"
    description: "Research data for Goal 1"
    priority: 1
    dependencies: []
    tool_name: "search_web"
    arguments:
      query: "research topic for goal 1"

  - id: "goal_2_research"
    type: "ToolTask"
    description: "Research data for Goal 2"
    priority: 1
    dependencies: []
    tool_name: "search_web"
    arguments:
      query: "research topic for goal 2"

  - id: "analyze_combined"
    type: "LLMTask"
    description: "Analyze combined research results"
    priority: 2
    dependencies: ["goal_1_research", "goal_2_research"]
    prompt_template: |
      Analyze these research results:
      Goal 1 Data: {{ results.goal_1_research.data }}
      Goal 2 Data: {{ results.goal_2_research.data }}

      Provide comprehensive analysis addressing both goals.
    llm_config:
      model_preference: "complex"
      temperature: 0.3
Generate the execution plan for the strategic goals:
    """

        else:
            # Legacy single-query planning
            base_query = prep_res['query']
            prompt = f"""
You are an expert task planner with dynamic adaptation capabilities.
Create intelligent, adaptive execution plans for the user query.
User Query
{base_query}
Your Available Tools & Intelligence
{tool_intelligence}
{variable_manager.get_llm_variable_context() if variable_manager else ""}
TASK TYPES (Dataclass-Aligned)

LLMTask: Step that uses a language model
ToolTask: Step that calls an available tool
DecisionTask: Step that decides routing between tasks

YAML SCHEMA
yamlCopyplan_name: string
description: string
execution_strategy: "sequential" | "parallel" | "mixed"
tasks:
  - id: string
    type: "LLMTask" | "ToolTask" | "DecisionTask"
    description: string
    priority: int
    dependencies: [list of task ids]
    # Additional fields depending on type
Generate the adaptive execution plan:
            """

        try:
            model_to_use = prep_res.get("complex_llm_model", "openrouter/openai/gpt-4o")
            agent_instance = prep_res["agent_instance"]

            plan_data = await agent_instance.a_format_class(PlanData,
                model=model_to_use,
                prompt= prompt,
                temperature=0.3,
                auto_context=True,
                node_name="TaskPlannerNode",
                task_id="goals_based_planning" if planning_mode == "goals_based" else "adaptive_planning"
            )
            # Create specialized tasks
            tasks = []
            for task_data in plan_data.get("tasks", []):
                task_type = task_data.pop("type", "generic")
                task = create_task(task_type, **task_data)
                tasks.append(task)

            plan = TaskPlan(
                id=str(uuid.uuid4()),
                name=plan_data.get("plan_name", "Generated Plan"),
                description=plan_data.get("description",
                                          "Plan for goals-based execution" if planning_mode == "goals_based" else f"Plan for: {base_query}"),
                tasks=tasks,
                execution_strategy=plan_data.get("execution_strategy", "sequential"),
                metadata={
                    "planning_mode": planning_mode,
                    "goals_count": len(prep_res.get("goals", [])) if planning_mode == "goals_based" else 1
                }
            )

            rprint(f"Created {planning_mode} plan with {len(tasks)} tasks")
            return plan

        except Exception as e:
            eprint(f"Advanced planning failed: {e}")
            import traceback

            print(traceback.format_exc())
            return await self._create_simple_plan(prep_res)

    def _build_tool_intelligence(self, prep_res: dict) -> str:
        """Build detailed tool intelligence for planning"""

        agent_instance = prep_res.get("agent_instance")
        if not agent_instance or not hasattr(agent_instance, '_tool_capabilities'):
            return "No tool intelligence available."

        capabilities = agent_instance._tool_capabilities
        query = prep_res.get('query', '').lower()

        context_parts = []
        context_parts.append("### Intelligent Tool Analysis:")

        for tool_name, cap in capabilities.items():
            context_parts.append(f"\n{tool_name}:")
            context_parts.append(f"- Function: {cap.get('primary_function', 'Unknown')}")
            context_parts.append(f"- Arguments: {yaml.dump(cap.get('args_schema', 'takes no arguments!'), default_flow_style=False)}")

            # Check relevance to current query
            relevance_score = self._calculate_tool_relevance(query, cap)
            context_parts.append(f"- Query relevance: {relevance_score:.2f}")

            if relevance_score > 0.4:
                context_parts.append("- ⭐ HIGHLY RELEVANT - SHOULD USE THIS TOOL!")

            # Show trigger analysis
            triggers = cap.get('trigger_phrases', [])
            matched_triggers = [t for t in triggers if t.lower() in query]
            if matched_triggers:
                context_parts.append(f"- Matched triggers: {matched_triggers}")

            # Show use cases
            use_cases = cap.get('use_cases', [])[:3]
            context_parts.append(f"- Use cases: {', '.join(use_cases)}")

        return "\n".join(context_parts)

    def _calculate_tool_relevance(self, query: str, capabilities: dict) -> float:
        """Calculate how relevant a tool is to the current query"""

        query_words = set(query.lower().split())

        # Check trigger phrases
        trigger_score = 0.0
        triggers = capabilities.get('trigger_phrases', [])
        for trigger in triggers:
            trigger_words = set(trigger.lower().split())
            if trigger_words.intersection(query_words):
                trigger_score += 0.04
        # Check confidence triggers if available
        conf_triggers = capabilities.get('confidence_triggers', {})
        for phrase, confidence in conf_triggers.items():
            if phrase.lower() in query:
                trigger_score += confidence/10
        # Check indirect connections
        indirect = capabilities.get('indirect_connections', [])
        for connection in indirect:
            connection_words = set(connection.lower().split())
            if connection_words.intersection(query_words):
                trigger_score += 0.02
        return min(1.0, trigger_score)
post_async(shared, prep_res, exec_res) async

Post-processing nach Plan-Erstellung

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
async def post_async(self, shared, prep_res, exec_res):
    """Post-processing nach Plan-Erstellung"""

    if exec_res is None:
        shared["planning_error"] = "Plan creation returned None"
        return "planning_failed"

    if isinstance(exec_res, TaskPlan):

        progress_tracker = shared.get("progress_tracker")
        if progress_tracker:
            await progress_tracker.emit_event(ProgressEvent(
                event_type="plan_created",
                node_name="TaskPlannerNode",
                session_id=shared.get("session_id"),
                status=NodeStatus.COMPLETED,
                success=True,
                plan_id=exec_res.id,
                metadata={
                    "plan_name": exec_res.name,
                    "task_count": len(exec_res.tasks),
                    "strategy": exec_res.execution_strategy
                }
            ))

        # Erfolgreicher Plan
        shared["current_plan"] = exec_res

        # Tasks in shared state für Executor verfügbar machen
        task_dict = {task.id: task for task in exec_res.tasks}
        if "tasks" not in shared:
            shared["tasks"] = task_dict
        else:
            shared["tasks"].update(task_dict)

        # Plan-Metadaten setzen
        shared["plan_created_at"] = datetime.now().isoformat()
        shared["plan_strategy"] = exec_res.execution_strategy
        shared["total_tasks_planned"] = len(exec_res.tasks)

        rprint(f"Plan created successfully: {exec_res.name} with {len(exec_res.tasks)} tasks")
        return "planned"

    else:
        # Plan creation failed
        shared["planning_error"] = "Invalid plan format returned"
        shared["current_plan"] = None
        eprint("Plan creation failed - invalid format")
        return "planning_failed"
prep_async(shared) async

Enhanced preparation with goals-based planning support

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
async def prep_async(self, shared):
    """Enhanced preparation with goals-based planning support"""

    # Check if this is a goals-based call from LLMReasonerNode
    replan_context = shared.get("replan_context", {})
    goals_list = replan_context.get("goals", [])

    if goals_list:
        # Goals-based planning (called by LLMReasonerNode)
        return {
            "goals": goals_list,
            "planning_mode": "goals_based",
            "query": shared.get("current_query", ""),
            "reasoning_context": replan_context.get("reasoning_context", ""),
            "triggered_by": replan_context.get("triggered_by", "unknown"),
            "tasks": shared.get("tasks", {}),
            "system_status": shared.get("system_status", "idle"),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "available_tools_names": shared.get("available_tools", []),
            "strategy": "goals_decomposition",  # New strategy type
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "agent_instance": shared.get("agent_instance"),
            "variable_manager": shared.get("variable_manager"),
        }
    else:
        # Legacy planning (original query-based approach)
        return {
            "query": shared.get("current_query", ""),
            "tasks": shared.get("tasks", {}),
            "system_status": shared.get("system_status", "idle"),
            "tool_capabilities": shared.get("tool_capabilities", {}),
            "available_tools_names": shared.get("available_tools", []),
            "strategy": shared.get("selected_strategy", "direct_response"),
            "fast_llm_model": shared.get("fast_llm_model"),
            "complex_llm_model": shared.get("complex_llm_model"),
            "agent_instance": shared.get("agent_instance"),
            "variable_manager": shared.get("variable_manager"),
            "planning_mode": "legacy"
        }
ToolAnalysis

Bases: BaseModel

Defines the structure for a valid tool analysis.

Source code in toolboxv2/mods/isaa/base/Agent/types.py
795
796
797
798
799
800
801
802
803
804
805
class ToolAnalysis(BaseModel):
    """Defines the structure for a valid tool analysis."""
    primary_function: str = Field(..., description="The main purpose of the tool.")
    use_cases: list[str] = Field(..., description="Specific use cases for the tool.")
    trigger_phrases: list[str] = Field(..., description="Phrases that should trigger the tool.")
    indirect_connections: list[str] = Field(..., description="Non-obvious connections or applications.")
    complexity_scenarios: list[str] = Field(..., description="Complex scenarios where the tool can be applied.")
    user_intent_categories: list[str] = Field(..., description="Categories of user intent the tool addresses.")
    confidence_triggers: dict[str, float] = Field(..., description="Phrases mapped to confidence scores.")
    tool_complexity: str = Field(..., description="The complexity of the tool, rated as low, medium, or high.")
    args_schema: dict[str, Any] | None = Field(..., description="The schema for the tool's arguments.")
ToolTask dataclass

Bases: Task

Spezialisierter Task für Tool-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py
488
489
490
491
492
493
494
495
@dataclass
class ToolTask(Task):
    """Spezialisierter Task für Tool-Aufrufe"""
    tool_name: str = ""
    arguments: dict[str, Any] = field(default_factory=dict)  # Kann {{ }} Referenzen enthalten
    hypothesis: str = ""  # Was erwarten wir von diesem Tool?
    validation_criteria: str = ""  # Wie validieren wir das Ergebnis?
    expectation: str = ""  # Wie sollte das Ergebnis aussehen?
UnifiedBindingManager

Unified manager that handles both shared and private variable scopes for bound agents

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11692
11693
11694
11695
11696
11697
11698
11699
11700
11701
11702
11703
11704
11705
11706
11707
11708
11709
11710
11711
11712
11713
11714
11715
11716
11717
11718
11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
11730
11731
11732
11733
11734
11735
11736
11737
11738
11739
11740
11741
11742
11743
11744
11745
11746
11747
11748
11749
11750
11751
11752
11753
11754
11755
11756
11757
11758
11759
11760
11761
11762
11763
11764
11765
11766
11767
11768
11769
11770
11771
11772
11773
11774
11775
11776
11777
11778
11779
11780
11781
class UnifiedBindingManager:
    """Unified manager that handles both shared and private variable scopes for bound agents"""

    def __init__(self, shared_manager: VariableManager, private_manager: VariableManager,
                 agent_name: str, shared_scopes: list[str], auto_sync: bool, binding_config: dict):
        self.shared_manager = shared_manager
        self.private_manager = private_manager
        self.agent_name = agent_name
        self.shared_scopes = shared_scopes
        self.auto_sync = auto_sync
        self.binding_config = binding_config

    def get(self, path: str, default=None, use_cache: bool = True):
        """Get variable from appropriate manager (shared or private)"""
        scope = path.split('.')[0] if '.' in path else path

        if scope in self.shared_scopes:
            return self.shared_manager.get(path, default, use_cache)
        else:
            # Try private first, then shared as fallback
            result = self.private_manager.get(path, None, use_cache)
            if result is None:
                return self.shared_manager.get(path, default, use_cache)
            return result

    def set(self, path: str, value, create_scope: bool = True):
        """Set variable in appropriate manager (shared or private)"""
        scope = path.split('.')[0] if '.' in path else path

        if scope in self.shared_scopes:
            self.shared_manager.set(path, value, create_scope)
            # Auto-sync to other bound agents if enabled
            if self.auto_sync:
                self._sync_to_bound_agents(path, value)
        else:
            # Private scope - add agent identifier
            private_path = f"{path}_{self.agent_name}" if not path.endswith(f"_{self.agent_name}") else path
            self.private_manager.set(private_path, value, create_scope)

    def _sync_to_bound_agents(self, path: str, value):
        """Sync shared variable changes to all bound agents"""
        try:
            bound_agents = self.binding_config.get('agents', [])
            for agent in bound_agents:
                if (agent.amd.name != self.agent_name and
                    hasattr(agent, 'variable_manager') and
                    isinstance(agent.variable_manager, UnifiedBindingManager)):
                    agent.variable_manager.shared_manager.set(path, value, create_scope=True)
        except Exception as e:
            wprint(f"Auto-sync failed for path {path}: {e}")

    def format_text(self, text: str, context: dict = None) -> str:
        """Format text with variables from both managers"""
        # First try private manager, then shared manager
        try:
            result = self.private_manager.format_text(text, context)
            return self.shared_manager.format_text(result, context)
        except:
            return self.shared_manager.format_text(text, context)

    def get_available_variables(self) -> dict[str, dict]:
        """Get available variables from both managers"""
        shared_vars = self.shared_manager.get_available_variables()
        private_vars = self.private_manager.get_available_variables()

        # Merge with prefix for private vars
        combined = shared_vars.copy()
        for key, value in private_vars.items():
            combined[f"private_{self.agent_name}_{key}"] = value

        return combined

    def get_scope_info(self) -> dict[str, Any]:
        """Get scope information from both managers"""
        shared_info = self.shared_manager.get_scope_info()
        private_info = self.private_manager.get_scope_info()

        return {
            'shared_scopes': shared_info,
            'private_scopes': private_info,
            'binding_info': {
                'agent_name': self.agent_name,
                'binding_id': self.binding_config.get('binding_id'),
                'auto_sync': self.auto_sync
            }
        }

    # Delegate other methods to shared manager by default
    def __getattr__(self, name):
        return getattr(self.shared_manager, name)
format_text(text, context=None)

Format text with variables from both managers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11743
11744
11745
11746
11747
11748
11749
11750
def format_text(self, text: str, context: dict = None) -> str:
    """Format text with variables from both managers"""
    # First try private manager, then shared manager
    try:
        result = self.private_manager.format_text(text, context)
        return self.shared_manager.format_text(result, context)
    except:
        return self.shared_manager.format_text(text, context)
get(path, default=None, use_cache=True)

Get variable from appropriate manager (shared or private)

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11704
11705
11706
11707
11708
11709
11710
11711
11712
11713
11714
11715
def get(self, path: str, default=None, use_cache: bool = True):
    """Get variable from appropriate manager (shared or private)"""
    scope = path.split('.')[0] if '.' in path else path

    if scope in self.shared_scopes:
        return self.shared_manager.get(path, default, use_cache)
    else:
        # Try private first, then shared as fallback
        result = self.private_manager.get(path, None, use_cache)
        if result is None:
            return self.shared_manager.get(path, default, use_cache)
        return result
get_available_variables()

Get available variables from both managers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11752
11753
11754
11755
11756
11757
11758
11759
11760
11761
11762
def get_available_variables(self) -> dict[str, dict]:
    """Get available variables from both managers"""
    shared_vars = self.shared_manager.get_available_variables()
    private_vars = self.private_manager.get_available_variables()

    # Merge with prefix for private vars
    combined = shared_vars.copy()
    for key, value in private_vars.items():
        combined[f"private_{self.agent_name}_{key}"] = value

    return combined
get_scope_info()

Get scope information from both managers

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11764
11765
11766
11767
11768
11769
11770
11771
11772
11773
11774
11775
11776
11777
def get_scope_info(self) -> dict[str, Any]:
    """Get scope information from both managers"""
    shared_info = self.shared_manager.get_scope_info()
    private_info = self.private_manager.get_scope_info()

    return {
        'shared_scopes': shared_info,
        'private_scopes': private_info,
        'binding_info': {
            'agent_name': self.agent_name,
            'binding_id': self.binding_config.get('binding_id'),
            'auto_sync': self.auto_sync
        }
    }
set(path, value, create_scope=True)

Set variable in appropriate manager (shared or private)

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11717
11718
11719
11720
11721
11722
11723
11724
11725
11726
11727
11728
11729
def set(self, path: str, value, create_scope: bool = True):
    """Set variable in appropriate manager (shared or private)"""
    scope = path.split('.')[0] if '.' in path else path

    if scope in self.shared_scopes:
        self.shared_manager.set(path, value, create_scope)
        # Auto-sync to other bound agents if enabled
        if self.auto_sync:
            self._sync_to_bound_agents(path, value)
    else:
        # Private scope - add agent identifier
        private_path = f"{path}_{self.agent_name}" if not path.endswith(f"_{self.agent_name}") else path
        self.private_manager.set(private_path, value, create_scope)
UnifiedContextManager

Zentrale Orchestrierung aller Context-Quellen für einheitlichen und effizienten Datenzugriff. Vereinigt ChatSession, VariableManager, World Model und Task Results.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7665
7666
7667
7668
7669
7670
7671
7672
7673
7674
7675
7676
7677
7678
7679
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
7720
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
7755
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
7785
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
7850
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
7979
7980
7981
7982
7983
7984
7985
7986
7987
7988
7989
7990
7991
7992
7993
7994
7995
7996
7997
7998
7999
8000
8001
8002
8003
8004
8005
8006
8007
8008
8009
8010
8011
8012
8013
8014
8015
8016
8017
8018
8019
8020
8021
8022
8023
8024
8025
8026
8027
8028
8029
8030
8031
8032
8033
8034
8035
8036
8037
8038
8039
8040
8041
8042
8043
8044
8045
8046
8047
8048
8049
8050
8051
8052
8053
8054
8055
8056
8057
8058
8059
8060
8061
8062
8063
8064
8065
8066
8067
8068
8069
8070
8071
8072
8073
8074
8075
8076
8077
8078
8079
8080
8081
8082
8083
8084
8085
8086
8087
8088
8089
8090
8091
8092
8093
8094
8095
8096
8097
8098
8099
8100
8101
8102
8103
8104
8105
8106
8107
8108
8109
8110
8111
8112
8113
8114
8115
8116
8117
8118
8119
8120
8121
8122
8123
8124
8125
8126
8127
8128
8129
8130
8131
8132
8133
8134
8135
8136
8137
8138
8139
8140
8141
8142
8143
8144
8145
8146
8147
8148
8149
8150
8151
8152
8153
8154
8155
8156
8157
8158
8159
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175
8176
8177
8178
8179
8180
8181
8182
8183
8184
8185
8186
class UnifiedContextManager:
    """
    Zentrale Orchestrierung aller Context-Quellen für einheitlichen und effizienten Datenzugriff.
    Vereinigt ChatSession, VariableManager, World Model und Task Results.
    """

    def __init__(self, agent):
        self.agent = agent
        self.session_managers: dict[str, Any] = {}  # ChatSession objects
        self.variable_manager: VariableManager = None
        self.compression_threshold = 15  # Messages before compression
        self._context_cache: dict[str, tuple[float, Any]] = {}  # (timestamp, data)
        self.cache_ttl = 300  # 5 minutes
        self._memory_instance = None

    async def initialize_session(self, session_id: str, max_history: int = 200):
        """Initialisiere oder lade existierende ChatSession als primäre Context-Quelle"""
        if session_id not in self.session_managers:
            try:
                # Get memory instance
                if not self._memory_instance:
                    from toolboxv2 import get_app
                    self._memory_instance = get_app().get_mod("isaa").get_memory()
                from toolboxv2.mods.isaa.extras.session import ChatSession
                # Create ChatSession as PRIMARY memory source
                session = ChatSession(
                    self._memory_instance,
                    max_length=max_history,
                    space_name=f"ChatSession/{self.agent.amd.name}.{session_id}.unified"
                )
                self.session_managers[session_id] = session

                # Integration mit VariableManager wenn verfügbar
                if self.variable_manager:
                    self.variable_manager.register_scope(f'session_{session_id}', {
                        'chat_session_active': True,
                        'history_length': len(session.history),
                        'last_interaction': None,
                        'session_id': session_id
                    })

                rprint(f"Unified session context initialized for {session_id}")
                return session

            except Exception as e:
                eprint(f"Failed to create ChatSession for {session_id}: {e}")
                # Fallback: Create minimal session manager
                self.session_managers[session_id] = {
                    'history': [],
                    'session_id': session_id,
                    'fallback_mode': True
                }
                return self.session_managers[session_id]

        return self.session_managers[session_id]

    async def add_interaction(self, session_id: str, role: str, content: str, metadata: dict = None) -> None:
        """Einheitlicher Weg um Interaktionen in ChatSession zu speichern"""
        session = await self.initialize_session(session_id)

        message = {
            'role': role,
            'content': content,
            'timestamp': datetime.now().isoformat(),
            'session_id': session_id,
            'metadata': metadata or {}
        }

        # PRIMARY: Store in ChatSession
        if hasattr(session, 'add_message'):
            from toolboxv2 import get_app
            get_app().run_bg_task_advanced(session.add_message, message, direct=False)
        elif isinstance(session, dict) and 'history' in session:
            # Fallback mode
            session['history'].append(message)
            # Keep max length
            max_len = 200
            if len(session['history']) > max_len:
                session['history'] = session['history'][-max_len:]

        # SECONDARY: Update VariableManager
        if self.variable_manager:
            self.variable_manager.set(f'session_{session_id}.last_interaction', message)
            if hasattr(session, 'history'):
                self.variable_manager.set(f'session_{session_id}.history_length', len(session.history))
            elif isinstance(session, dict):
                self.variable_manager.set(f'session_{session_id}.history_length', len(session.get('history', [])))

        # Clear context cache for this session
        self._invalidate_cache(session_id)

    async def get_contextual_history(self, session_id: str, query: str = "", max_entries: int = 10) -> list[dict]:
        """Intelligente Auswahl relevanter Geschichte aus ChatSession"""
        session = self.session_managers.get(session_id)
        if not session:
            return []

        try:
            # ChatSession mode
            if hasattr(session, 'get_past_x'):
                recent_history = session.get_past_x(max_entries, last_u=False)
                c = await session.get_reference(query)
                return recent_history[:max_entries] + ([] if not c else  [{'role': 'system', 'content': c,
                                                        'timestamp': datetime.now().isoformat(), 'metadata': {'source': 'contextual_history'}}] )

            # Fallback mode
            elif isinstance(session, dict) and 'history' in session:
                history = session['history']
                # Return last max_entries, starting with last user message
                result = []
                for msg in reversed(history[-max_entries:]):
                    result.append(msg)
                    if msg.get('role') == 'user' and len(result) >= max_entries:
                        break
                return list(reversed(result))[:max_entries]

        except Exception as e:
            eprint(f"Error getting contextual history: {e}")

        return []

    async def build_unified_context(self, session_id: str, query: str = None, context_type: str = "full") -> dict[
        str, Any]:
        """ZENTRALE Methode für vollständigen Context-Aufbau aus allen Quellen"""

        # Cache check
        cache_key = f"{session_id}_{hash(query or '')}_{context_type}"
        cached = self._get_cached_context(cache_key)
        if cached:
            return cached

        context: dict[str, Any] = {
            'timestamp': datetime.now().isoformat(),
            'session_id': session_id,
            'query': query,
            'context_type': context_type
        }

        try:
            # 1. CHAT HISTORY (Primary - from ChatSession)
            with Spinner("Building unified context..."):
                context['chat_history'] = await self.get_contextual_history(
                    session_id, query or "", max_entries=15
                )

            # 2. VARIABLE SYSTEM STATE
            if self.variable_manager:
                context['variables'] = {
                    'available_scopes': list(self.variable_manager.scopes.keys()),
                    'total_variables': len(self.variable_manager.get_available_variables()),
                    'recent_results': self._get_recent_results(5)
                }
            else:
                context['variables'] = {'status': 'variable_manager_not_available'}

            # 3. WORLD MODEL FACTS
            if self.variable_manager:
                world_model = self.variable_manager.get('world', {})
                if world_model and query:
                    context['relevant_facts'] = self._extract_relevant_facts(world_model, query)
                else:
                    context['relevant_facts'] = list(world_model.items())[:5]  # Top 5 facts

            # 4. EXECUTION STATE
            context['execution_state'] = {
                'active_tasks': self._get_active_tasks(),
                'recent_completions': self._get_recent_completions(3),
                'system_status': self.agent.shared.get('system_status', 'idle')
            }

            # 5. SESSION STATISTICS
            context['session_stats'] = {
                'total_sessions': len(self.session_managers),
                'current_session_length': len(context['chat_history']),
                'cache_enabled': bool(self._context_cache)
            }

        except Exception as e:
            eprint(f"Error building unified context: {e}")
            context['error'] = str(e)
            context['fallback_mode'] = True

        # Cache result
        self._cache_context(cache_key, context)
        return context

    def get_formatted_context_for_llm(self, unified_context: dict[str, Any]) -> str:
        """Formatiere unified context für LLM consumption"""
        try:
            parts = []

            # Header with session info
            session_id = unified_context.get('session_id', 'unknown')
            query = unified_context.get('query', '')
            context_type = unified_context.get('context_type', 'full')
            parts.append(f"## Session Context ({context_type})")
            parts.append(f"Session: {session_id}")
            if query:
                parts.append(f"Query: {query}")

            # Recent Chat History
            chat_history = unified_context.get('chat_history', [])
            if chat_history:
                parts.append("\n## Recent Conversation")
                for msg in chat_history[-5:]:  # Last 5 messages
                    timestamp = msg.get('timestamp', '')[:19]  # Remove microseconds
                    role = msg.get('role', 'unknown')
                    content = msg.get('content', '')
                    content_preview = content[:500] + ("..." if len(content) > 500 else "")
                    parts.append(f"[{timestamp}] {role}: {content_preview}")

            # Variable System State
            variables = unified_context.get('variables', {})
            if variables and variables != {'status': 'variable_manager_not_available'}:
                parts.append("\n## Variable System")

                # Available scopes
                scopes = variables.get('available_scopes', [])
                if scopes:
                    parts.append(f"Available Scopes: {', '.join(scopes)}")

                # Total variables count
                total_vars = variables.get('total_variables', 0)
                if total_vars > 0:
                    parts.append(f"Total Variables: {total_vars}")
                    parts.append(f"Total Variables Values: \n{yaml.dump(self.variable_manager.get_available_variables())}")

                # Recent results
                recent_results = variables.get('recent_results', [])
                if recent_results:
                    parts.append(f"Recent Results ({len(recent_results)}):")
                    for result in recent_results[:3]:  # Top 3 results
                        task_id = result.get('task_id', 'unknown')
                        preview = str(result.get('preview', ''))[:100]
                        preview += "..." if len(str(result.get('preview', ''))) > 100 else ""
                        parts.append(f"  - {task_id}: {preview}")
            elif variables.get('status') == 'variable_manager_not_available':
                parts.append("\n## Variable System: Not Available")

            # World Model Facts (Relevant Facts)
            relevant_facts = unified_context.get('relevant_facts', [])
            if relevant_facts:
                parts.append("\n## World Model Facts")
                for fact in relevant_facts[:5]:  # Top 5 facts
                    if isinstance(fact, (list, tuple)) and len(fact) >= 2:
                        # Handle (key, value) tuple format
                        key, value = fact[0], fact[1]
                        fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                        parts.append(f"- {key}: {fact_preview}")
                    elif isinstance(fact, dict):
                        # Handle dict format
                        for key, value in list(fact.items())[:1]:  # Just first item
                            fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                            parts.append(f"- {key}: {fact_preview}")

            # Execution State
            execution_state = unified_context.get('execution_state', {})
            if execution_state:
                parts.append("\n## System Status")

                system_status = execution_state.get('system_status', 'unknown')
                parts.append(f"Status: {system_status}")

                active_tasks = execution_state.get('active_tasks', [])
                if active_tasks:
                    parts.append(f"Active Tasks: {len(active_tasks)}")
                    # Show task previews if available
                    for task in active_tasks[:2]:  # Show first 2 tasks
                        task_str = str(task)[:80] + ("..." if len(str(task)) > 80 else "")
                        parts.append(f"  - {task_str}")

                recent_completions = execution_state.get('recent_completions', [])
                if recent_completions:
                    parts.append(f"Recent Completions: {len(recent_completions)}")
                    # Show completion previews if available
                    for completion in recent_completions[:2]:  # Show first 2 completions
                        comp_str = str(completion)[:80] + ("..." if len(str(completion)) > 80 else "")
                        parts.append(f"  - {comp_str}")

            # Session Statistics
            session_stats = unified_context.get('session_stats', {})
            if session_stats:
                parts.append("\n## Session Statistics")

                total_sessions = session_stats.get('total_sessions', 0)
                if total_sessions > 0:
                    parts.append(f"Total Sessions: {total_sessions}")

                current_length = session_stats.get('current_session_length', 0)
                if current_length > 0:
                    parts.append(f"Current Session Length: {current_length} messages")

                cache_enabled = session_stats.get('cache_enabled', False)
                parts.append(f"Cache Enabled: {cache_enabled}")

            # Error handling
            if unified_context.get('error'):
                parts.append(f"\n## Error")
                parts.append(f"Error: {unified_context['error']}")

            if unified_context.get('fallback_mode'):
                parts.append("⚠️  Running in fallback mode")

            # Footer with timestamp
            timestamp = unified_context.get('timestamp', 'unknown')
            parts.append(f"\n---\nContext generated at: {timestamp}")

            return "\n".join(parts)

        except Exception as e:
            eprint(f"Error formatting context for LLM: {e}")
            import traceback
            print(traceback.format_exc())
            return f"Context formatting error: {str(e)}"

    def _merge_and_dedupe_history(self, recent_history: list[dict], relevant_refs: list) -> list[dict]:
        """Merge und dedupliziere History-Einträge"""
        try:
            merged = recent_history.copy()

            # Add relevant references if they're not already in recent history
            for ref in relevant_refs:
                # Convert ref to message format if needed
                if isinstance(ref, dict) and 'content' in ref:
                    # Check if not already in recent_history
                    is_duplicate = any(
                        msg.get('content', '') == ref.get('content', '') and
                        msg.get('timestamp', '') == ref.get('timestamp', '')
                        for msg in merged
                    )
                    if not is_duplicate:
                        merged.append(ref)

            # Sort by timestamp
            merged.sort(key=lambda x: x.get('timestamp', ''))

            return merged
        except:
            return recent_history

    def _get_recent_results(self, limit: int = 5) -> list[dict]:
        """Hole recent results aus dem shared state"""
        try:
            results_store = self.agent.shared.get("results", {})
            recent_results = []

            for task_id, result_data in list(results_store.items())[-limit:]:
                if result_data and result_data.get("data"):
                    preview = str(result_data["data"])[:150] + "..."
                    recent_results.append({
                        "task_id": task_id,
                        "preview": preview,
                        "success": result_data.get("metadata", {}).get("success", False),
                        "timestamp": result_data.get("metadata", {}).get("completed_at")
                    })

            return recent_results
        except:
            return []

    def _extract_relevant_facts(self, world_model: dict, query: str) -> list[tuple[str, Any]]:
        """Extrahiere relevante Facts basierend auf Query"""
        try:
            query_words = set(query.lower().split())
            relevant_facts = []

            for key, value in world_model.items():
                # Simple relevance scoring
                key_words = set(key.lower().split())
                value_words = set(str(value).lower().split())

                # Check for word overlap
                key_overlap = len(query_words.intersection(key_words))
                value_overlap = len(query_words.intersection(value_words))

                if key_overlap > 0 or value_overlap > 0:
                    relevance_score = key_overlap * 2 + value_overlap  # Key matches weighted higher
                    relevant_facts.append((relevance_score, key, value))

            # Sort by relevance and return top facts
            relevant_facts.sort(key=lambda x: x[0], reverse=True)
            return [(key, value) for _, key, value in relevant_facts[:5]]
        except:
            return list(world_model.items())[:5]

    def _get_active_tasks(self) -> list[dict]:
        """Hole aktive Tasks"""
        try:
            tasks = self.agent.shared.get("tasks", {})
            return [
                {"id": task_id, "description": task.description, "status": task.status}
                for task_id, task in tasks.items()
                if task.status == "running"
            ]
        except:
            return []

    def _get_recent_completions(self, limit: int = 3) -> list[dict]:
        """Hole recent completions"""
        try:
            tasks = self.agent.shared.get("tasks", {})
            completed = [
                {"id": task_id, "description": task.description, "completed_at": task.completed_at}
                for task_id, task in tasks.items()
                if task.status == "completed" and hasattr(task, 'completed_at') and task.completed_at
            ]
            # Sort by completion time
            completed.sort(key=lambda x: x.get('completed_at', ''), reverse=True)
            return completed[:limit]
        except:
            return []

    def _get_cached_context(self, cache_key: str) -> dict[str, Any] | None:
        """Hole Context aus Cache wenn noch gültig"""
        if cache_key in self._context_cache:
            timestamp, data = self._context_cache[cache_key]
            if time.time() - timestamp < self.cache_ttl:
                return data
            else:
                del self._context_cache[cache_key]
        return None

    def _cache_context(self, cache_key: str, context: dict[str, Any]):
        """Speichere Context in Cache"""
        self._context_cache[cache_key] = (time.time(), context.copy())

        # Cleanup old cache entries
        if len(self._context_cache) > 50:  # Keep max 50 entries
            oldest_key = min(self._context_cache.keys(),
                             key=lambda k: self._context_cache[k][0])
            del self._context_cache[oldest_key]

    def _invalidate_cache(self, session_id: str = None):
        """Invalidate cache for specific session or all"""
        if session_id:
            # Remove all cache entries for this session
            keys_to_remove = [k for k in self._context_cache if session_id in k]
            for key in keys_to_remove:
                del self._context_cache[key]
        else:
            self._context_cache.clear()

    def get_session_statistics(self) -> dict[str, Any]:
        """Hole Statistiken über alle Sessions"""
        stats = {
            "total_sessions": len(self.session_managers),
            "active_sessions": [],
            "cache_entries": len(self._context_cache),
            "cache_hit_rate": 0.0  # Could be tracked if needed
        }

        for session_id, session in self.session_managers.items():
            session_info = {
                "session_id": session_id,
                "fallback_mode": isinstance(session, dict) and session.get('fallback_mode', False)
            }

            if hasattr(session, 'history'):
                session_info["message_count"] = len(session.history)
            elif isinstance(session, dict) and 'history' in session:
                session_info["message_count"] = len(session['history'])

            stats["active_sessions"].append(session_info)

        return stats

    async def cleanup_old_sessions(self, max_age_hours: int = 168) -> int:
        """Cleanup alte Sessions (default: 1 Woche)"""
        try:
            cutoff_time = datetime.now() - timedelta(hours=max_age_hours)
            removed_count = 0

            sessions_to_remove = []
            for session_id, session in self.session_managers.items():
                should_remove = False

                # Check last activity
                if hasattr(session, 'history') and session.history:
                    last_msg = session.history[-1]
                    last_timestamp = last_msg.get('timestamp')
                    if last_timestamp:
                        try:
                            last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                            if last_time < cutoff_time:
                                should_remove = True
                        except:
                            pass
                elif isinstance(session, dict) and session.get('history'):
                    last_msg = session['history'][-1]
                    last_timestamp = last_msg.get('timestamp')
                    if last_timestamp:
                        try:
                            last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                            if last_time < cutoff_time:
                                should_remove = True
                        except:
                            pass

                if should_remove:
                    sessions_to_remove.append(session_id)

            # Remove old sessions
            for session_id in sessions_to_remove:
                session = self.session_managers[session_id]
                if hasattr(session, 'on_exit'):
                    session.on_exit()  # Save ChatSession data
                del self.session_managers[session_id]
                removed_count += 1

                # Remove from variable manager
                if self.variable_manager:
                    scope_name = f'session_{session_id}'
                    if scope_name in self.variable_manager.scopes:
                        del self.variable_manager.scopes[scope_name]

            # Clear related cache entries
            self._invalidate_cache()

            return removed_count
        except Exception as e:
            eprint(f"Error cleaning up old sessions: {e}")
            return 0
add_interaction(session_id, role, content, metadata=None) async

Einheitlicher Weg um Interaktionen in ChatSession zu speichern

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7721
7722
7723
7724
7725
7726
7727
7728
7729
7730
7731
7732
7733
7734
7735
7736
7737
7738
7739
7740
7741
7742
7743
7744
7745
7746
7747
7748
7749
7750
7751
7752
7753
7754
async def add_interaction(self, session_id: str, role: str, content: str, metadata: dict = None) -> None:
    """Einheitlicher Weg um Interaktionen in ChatSession zu speichern"""
    session = await self.initialize_session(session_id)

    message = {
        'role': role,
        'content': content,
        'timestamp': datetime.now().isoformat(),
        'session_id': session_id,
        'metadata': metadata or {}
    }

    # PRIMARY: Store in ChatSession
    if hasattr(session, 'add_message'):
        from toolboxv2 import get_app
        get_app().run_bg_task_advanced(session.add_message, message, direct=False)
    elif isinstance(session, dict) and 'history' in session:
        # Fallback mode
        session['history'].append(message)
        # Keep max length
        max_len = 200
        if len(session['history']) > max_len:
            session['history'] = session['history'][-max_len:]

    # SECONDARY: Update VariableManager
    if self.variable_manager:
        self.variable_manager.set(f'session_{session_id}.last_interaction', message)
        if hasattr(session, 'history'):
            self.variable_manager.set(f'session_{session_id}.history_length', len(session.history))
        elif isinstance(session, dict):
            self.variable_manager.set(f'session_{session_id}.history_length', len(session.get('history', [])))

    # Clear context cache for this session
    self._invalidate_cache(session_id)
build_unified_context(session_id, query=None, context_type='full') async

ZENTRALE Methode für vollständigen Context-Aufbau aus allen Quellen

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7786
7787
7788
7789
7790
7791
7792
7793
7794
7795
7796
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810
7811
7812
7813
7814
7815
7816
7817
7818
7819
7820
7821
7822
7823
7824
7825
7826
7827
7828
7829
7830
7831
7832
7833
7834
7835
7836
7837
7838
7839
7840
7841
7842
7843
7844
7845
7846
7847
7848
7849
async def build_unified_context(self, session_id: str, query: str = None, context_type: str = "full") -> dict[
    str, Any]:
    """ZENTRALE Methode für vollständigen Context-Aufbau aus allen Quellen"""

    # Cache check
    cache_key = f"{session_id}_{hash(query or '')}_{context_type}"
    cached = self._get_cached_context(cache_key)
    if cached:
        return cached

    context: dict[str, Any] = {
        'timestamp': datetime.now().isoformat(),
        'session_id': session_id,
        'query': query,
        'context_type': context_type
    }

    try:
        # 1. CHAT HISTORY (Primary - from ChatSession)
        with Spinner("Building unified context..."):
            context['chat_history'] = await self.get_contextual_history(
                session_id, query or "", max_entries=15
            )

        # 2. VARIABLE SYSTEM STATE
        if self.variable_manager:
            context['variables'] = {
                'available_scopes': list(self.variable_manager.scopes.keys()),
                'total_variables': len(self.variable_manager.get_available_variables()),
                'recent_results': self._get_recent_results(5)
            }
        else:
            context['variables'] = {'status': 'variable_manager_not_available'}

        # 3. WORLD MODEL FACTS
        if self.variable_manager:
            world_model = self.variable_manager.get('world', {})
            if world_model and query:
                context['relevant_facts'] = self._extract_relevant_facts(world_model, query)
            else:
                context['relevant_facts'] = list(world_model.items())[:5]  # Top 5 facts

        # 4. EXECUTION STATE
        context['execution_state'] = {
            'active_tasks': self._get_active_tasks(),
            'recent_completions': self._get_recent_completions(3),
            'system_status': self.agent.shared.get('system_status', 'idle')
        }

        # 5. SESSION STATISTICS
        context['session_stats'] = {
            'total_sessions': len(self.session_managers),
            'current_session_length': len(context['chat_history']),
            'cache_enabled': bool(self._context_cache)
        }

    except Exception as e:
        eprint(f"Error building unified context: {e}")
        context['error'] = str(e)
        context['fallback_mode'] = True

    # Cache result
    self._cache_context(cache_key, context)
    return context
cleanup_old_sessions(max_age_hours=168) async

Cleanup alte Sessions (default: 1 Woche)

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8131
8132
8133
8134
8135
8136
8137
8138
8139
8140
8141
8142
8143
8144
8145
8146
8147
8148
8149
8150
8151
8152
8153
8154
8155
8156
8157
8158
8159
8160
8161
8162
8163
8164
8165
8166
8167
8168
8169
8170
8171
8172
8173
8174
8175
8176
8177
8178
8179
8180
8181
8182
8183
8184
8185
8186
async def cleanup_old_sessions(self, max_age_hours: int = 168) -> int:
    """Cleanup alte Sessions (default: 1 Woche)"""
    try:
        cutoff_time = datetime.now() - timedelta(hours=max_age_hours)
        removed_count = 0

        sessions_to_remove = []
        for session_id, session in self.session_managers.items():
            should_remove = False

            # Check last activity
            if hasattr(session, 'history') and session.history:
                last_msg = session.history[-1]
                last_timestamp = last_msg.get('timestamp')
                if last_timestamp:
                    try:
                        last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                        if last_time < cutoff_time:
                            should_remove = True
                    except:
                        pass
            elif isinstance(session, dict) and session.get('history'):
                last_msg = session['history'][-1]
                last_timestamp = last_msg.get('timestamp')
                if last_timestamp:
                    try:
                        last_time = datetime.fromisoformat(last_timestamp.replace('Z', '+00:00'))
                        if last_time < cutoff_time:
                            should_remove = True
                    except:
                        pass

            if should_remove:
                sessions_to_remove.append(session_id)

        # Remove old sessions
        for session_id in sessions_to_remove:
            session = self.session_managers[session_id]
            if hasattr(session, 'on_exit'):
                session.on_exit()  # Save ChatSession data
            del self.session_managers[session_id]
            removed_count += 1

            # Remove from variable manager
            if self.variable_manager:
                scope_name = f'session_{session_id}'
                if scope_name in self.variable_manager.scopes:
                    del self.variable_manager.scopes[scope_name]

        # Clear related cache entries
        self._invalidate_cache()

        return removed_count
    except Exception as e:
        eprint(f"Error cleaning up old sessions: {e}")
        return 0
get_contextual_history(session_id, query='', max_entries=10) async

Intelligente Auswahl relevanter Geschichte aus ChatSession

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7756
7757
7758
7759
7760
7761
7762
7763
7764
7765
7766
7767
7768
7769
7770
7771
7772
7773
7774
7775
7776
7777
7778
7779
7780
7781
7782
7783
7784
async def get_contextual_history(self, session_id: str, query: str = "", max_entries: int = 10) -> list[dict]:
    """Intelligente Auswahl relevanter Geschichte aus ChatSession"""
    session = self.session_managers.get(session_id)
    if not session:
        return []

    try:
        # ChatSession mode
        if hasattr(session, 'get_past_x'):
            recent_history = session.get_past_x(max_entries, last_u=False)
            c = await session.get_reference(query)
            return recent_history[:max_entries] + ([] if not c else  [{'role': 'system', 'content': c,
                                                    'timestamp': datetime.now().isoformat(), 'metadata': {'source': 'contextual_history'}}] )

        # Fallback mode
        elif isinstance(session, dict) and 'history' in session:
            history = session['history']
            # Return last max_entries, starting with last user message
            result = []
            for msg in reversed(history[-max_entries:]):
                result.append(msg)
                if msg.get('role') == 'user' and len(result) >= max_entries:
                    break
            return list(reversed(result))[:max_entries]

    except Exception as e:
        eprint(f"Error getting contextual history: {e}")

    return []
get_formatted_context_for_llm(unified_context)

Formatiere unified context für LLM consumption

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7851
7852
7853
7854
7855
7856
7857
7858
7859
7860
7861
7862
7863
7864
7865
7866
7867
7868
7869
7870
7871
7872
7873
7874
7875
7876
7877
7878
7879
7880
7881
7882
7883
7884
7885
7886
7887
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905
7906
7907
7908
7909
7910
7911
7912
7913
7914
7915
7916
7917
7918
7919
7920
7921
7922
7923
7924
7925
7926
7927
7928
7929
7930
7931
7932
7933
7934
7935
7936
7937
7938
7939
7940
7941
7942
7943
7944
7945
7946
7947
7948
7949
7950
7951
7952
7953
7954
7955
7956
7957
7958
7959
7960
7961
7962
7963
7964
7965
7966
7967
7968
7969
7970
7971
7972
7973
7974
7975
7976
7977
7978
def get_formatted_context_for_llm(self, unified_context: dict[str, Any]) -> str:
    """Formatiere unified context für LLM consumption"""
    try:
        parts = []

        # Header with session info
        session_id = unified_context.get('session_id', 'unknown')
        query = unified_context.get('query', '')
        context_type = unified_context.get('context_type', 'full')
        parts.append(f"## Session Context ({context_type})")
        parts.append(f"Session: {session_id}")
        if query:
            parts.append(f"Query: {query}")

        # Recent Chat History
        chat_history = unified_context.get('chat_history', [])
        if chat_history:
            parts.append("\n## Recent Conversation")
            for msg in chat_history[-5:]:  # Last 5 messages
                timestamp = msg.get('timestamp', '')[:19]  # Remove microseconds
                role = msg.get('role', 'unknown')
                content = msg.get('content', '')
                content_preview = content[:500] + ("..." if len(content) > 500 else "")
                parts.append(f"[{timestamp}] {role}: {content_preview}")

        # Variable System State
        variables = unified_context.get('variables', {})
        if variables and variables != {'status': 'variable_manager_not_available'}:
            parts.append("\n## Variable System")

            # Available scopes
            scopes = variables.get('available_scopes', [])
            if scopes:
                parts.append(f"Available Scopes: {', '.join(scopes)}")

            # Total variables count
            total_vars = variables.get('total_variables', 0)
            if total_vars > 0:
                parts.append(f"Total Variables: {total_vars}")
                parts.append(f"Total Variables Values: \n{yaml.dump(self.variable_manager.get_available_variables())}")

            # Recent results
            recent_results = variables.get('recent_results', [])
            if recent_results:
                parts.append(f"Recent Results ({len(recent_results)}):")
                for result in recent_results[:3]:  # Top 3 results
                    task_id = result.get('task_id', 'unknown')
                    preview = str(result.get('preview', ''))[:100]
                    preview += "..." if len(str(result.get('preview', ''))) > 100 else ""
                    parts.append(f"  - {task_id}: {preview}")
        elif variables.get('status') == 'variable_manager_not_available':
            parts.append("\n## Variable System: Not Available")

        # World Model Facts (Relevant Facts)
        relevant_facts = unified_context.get('relevant_facts', [])
        if relevant_facts:
            parts.append("\n## World Model Facts")
            for fact in relevant_facts[:5]:  # Top 5 facts
                if isinstance(fact, (list, tuple)) and len(fact) >= 2:
                    # Handle (key, value) tuple format
                    key, value = fact[0], fact[1]
                    fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                    parts.append(f"- {key}: {fact_preview}")
                elif isinstance(fact, dict):
                    # Handle dict format
                    for key, value in list(fact.items())[:1]:  # Just first item
                        fact_preview = str(value)[:100] + ("..." if len(str(value)) > 100 else "")
                        parts.append(f"- {key}: {fact_preview}")

        # Execution State
        execution_state = unified_context.get('execution_state', {})
        if execution_state:
            parts.append("\n## System Status")

            system_status = execution_state.get('system_status', 'unknown')
            parts.append(f"Status: {system_status}")

            active_tasks = execution_state.get('active_tasks', [])
            if active_tasks:
                parts.append(f"Active Tasks: {len(active_tasks)}")
                # Show task previews if available
                for task in active_tasks[:2]:  # Show first 2 tasks
                    task_str = str(task)[:80] + ("..." if len(str(task)) > 80 else "")
                    parts.append(f"  - {task_str}")

            recent_completions = execution_state.get('recent_completions', [])
            if recent_completions:
                parts.append(f"Recent Completions: {len(recent_completions)}")
                # Show completion previews if available
                for completion in recent_completions[:2]:  # Show first 2 completions
                    comp_str = str(completion)[:80] + ("..." if len(str(completion)) > 80 else "")
                    parts.append(f"  - {comp_str}")

        # Session Statistics
        session_stats = unified_context.get('session_stats', {})
        if session_stats:
            parts.append("\n## Session Statistics")

            total_sessions = session_stats.get('total_sessions', 0)
            if total_sessions > 0:
                parts.append(f"Total Sessions: {total_sessions}")

            current_length = session_stats.get('current_session_length', 0)
            if current_length > 0:
                parts.append(f"Current Session Length: {current_length} messages")

            cache_enabled = session_stats.get('cache_enabled', False)
            parts.append(f"Cache Enabled: {cache_enabled}")

        # Error handling
        if unified_context.get('error'):
            parts.append(f"\n## Error")
            parts.append(f"Error: {unified_context['error']}")

        if unified_context.get('fallback_mode'):
            parts.append("⚠️  Running in fallback mode")

        # Footer with timestamp
        timestamp = unified_context.get('timestamp', 'unknown')
        parts.append(f"\n---\nContext generated at: {timestamp}")

        return "\n".join(parts)

    except Exception as e:
        eprint(f"Error formatting context for LLM: {e}")
        import traceback
        print(traceback.format_exc())
        return f"Context formatting error: {str(e)}"
get_session_statistics()

Hole Statistiken über alle Sessions

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
8107
8108
8109
8110
8111
8112
8113
8114
8115
8116
8117
8118
8119
8120
8121
8122
8123
8124
8125
8126
8127
8128
8129
def get_session_statistics(self) -> dict[str, Any]:
    """Hole Statistiken über alle Sessions"""
    stats = {
        "total_sessions": len(self.session_managers),
        "active_sessions": [],
        "cache_entries": len(self._context_cache),
        "cache_hit_rate": 0.0  # Could be tracked if needed
    }

    for session_id, session in self.session_managers.items():
        session_info = {
            "session_id": session_id,
            "fallback_mode": isinstance(session, dict) and session.get('fallback_mode', False)
        }

        if hasattr(session, 'history'):
            session_info["message_count"] = len(session.history)
        elif isinstance(session, dict) and 'history' in session:
            session_info["message_count"] = len(session['history'])

        stats["active_sessions"].append(session_info)

    return stats
initialize_session(session_id, max_history=200) async

Initialisiere oder lade existierende ChatSession als primäre Context-Quelle

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7680
7681
7682
7683
7684
7685
7686
7687
7688
7689
7690
7691
7692
7693
7694
7695
7696
7697
7698
7699
7700
7701
7702
7703
7704
7705
7706
7707
7708
7709
7710
7711
7712
7713
7714
7715
7716
7717
7718
7719
async def initialize_session(self, session_id: str, max_history: int = 200):
    """Initialisiere oder lade existierende ChatSession als primäre Context-Quelle"""
    if session_id not in self.session_managers:
        try:
            # Get memory instance
            if not self._memory_instance:
                from toolboxv2 import get_app
                self._memory_instance = get_app().get_mod("isaa").get_memory()
            from toolboxv2.mods.isaa.extras.session import ChatSession
            # Create ChatSession as PRIMARY memory source
            session = ChatSession(
                self._memory_instance,
                max_length=max_history,
                space_name=f"ChatSession/{self.agent.amd.name}.{session_id}.unified"
            )
            self.session_managers[session_id] = session

            # Integration mit VariableManager wenn verfügbar
            if self.variable_manager:
                self.variable_manager.register_scope(f'session_{session_id}', {
                    'chat_session_active': True,
                    'history_length': len(session.history),
                    'last_interaction': None,
                    'session_id': session_id
                })

            rprint(f"Unified session context initialized for {session_id}")
            return session

        except Exception as e:
            eprint(f"Failed to create ChatSession for {session_id}: {e}")
            # Fallback: Create minimal session manager
            self.session_managers[session_id] = {
                'history': [],
                'session_id': session_id,
                'fallback_mode': True
            }
            return self.session_managers[session_id]

    return self.session_managers[session_id]
VariableManager

Unified variable management system with advanced features

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205
7206
7207
7208
7209
7210
7211
7212
7213
7214
7215
7216
7217
7218
7219
7220
7221
7222
7223
7224
7225
7226
7227
7228
7229
7230
7231
7232
7233
7234
7235
7236
7237
7238
7239
7240
7241
7242
7243
7244
7245
7246
7247
7248
7249
7250
7251
7252
7253
7254
7255
7256
7257
7258
7259
7260
7261
7262
7263
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
7277
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
7357
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
7383
7384
7385
7386
7387
7388
7389
7390
7391
7392
7393
7394
7395
7396
7397
7398
7399
7400
7401
7402
7403
7404
7405
7406
7407
7408
7409
7410
7411
7412
7413
7414
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428
7429
7430
7431
7432
7433
7434
7435
7436
7437
7438
7439
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
7463
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
7480
7481
7482
7483
7484
7485
7486
7487
7488
7489
7490
7491
7492
7493
7494
7495
7496
7497
7498
7499
7500
7501
7502
7503
7504
7505
7506
7507
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
7532
7533
7534
7535
7536
7537
7538
7539
7540
7541
7542
7543
7544
7545
7546
7547
7548
7549
7550
7551
7552
7553
7554
7555
7556
7557
7558
7559
7560
7561
7562
7563
7564
7565
7566
7567
7568
7569
7570
7571
7572
7573
7574
7575
7576
7577
7578
7579
7580
7581
7582
7583
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
7614
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
class VariableManager:
    """Unified variable management system with advanced features"""

    def __init__(self, world_model: dict, shared_state: dict = None):
        self.world_model = world_model
        self.shared_state = shared_state or {}
        self.scopes = {
            'world': world_model,
            'shared': self.shared_state,
            'results': {},
            'tasks': {},
            'user': {},
            'system': {}
        }
        self._cache = {}

    def register_scope(self, name: str, data: dict):
        """Register a new variable scope"""
        self.scopes[name] = data
        self._cache.clear()

    def set_results_store(self, results_store: dict):
        """Set the results store for task result references"""
        self.scopes['results'] = results_store
        self._cache.clear()

    def set_tasks_store(self, tasks_store: dict):
        """Set tasks store for task metadata access"""
        self.scopes['tasks'] = tasks_store
        self._cache.clear()

    def _resolve_path(self, path: str):
        """
        Internal helper to navigate a path that can contain both
        dictionary keys and list indices.
        """
        parts = path.split('.')

        # Determine the starting point
        if len(parts) == 1:
            # Simple key in the top-level world_model
            current = self.world_model
        else:
            scope_name = parts[0]
            if scope_name not in self.scopes:
                raise KeyError(f"Scope '{scope_name}' not found")
            current = self.scopes[scope_name]
            parts = parts[1:]  # Continue with the rest of the path

        # Navigate through the parts
        for part in parts:
            if isinstance(current, list):
                try:
                    # It's a list, so the part must be an integer index
                    index = int(part)
                    current = current[index]
                except (ValueError, IndexError):
                    raise KeyError(f"Invalid list index '{part}' in path '{path}'")
            elif isinstance(current, dict):
                try:
                    # It's a dictionary, so the part is a key
                    current = current[part]
                except KeyError:
                    raise KeyError(f"Key '{part}' not found in path '{path}'")
            else:
                # We've hit a non-collection type (int, str, etc.) but the path continues
                raise KeyError(f"Path cannot descend into non-collection type at '{part}' in path '{path}'")

        return current

    def get(self, path: str, default=None, use_cache: bool = True):
        """Get variable with dot notation path support for dicts and lists."""
        if use_cache and path in self._cache:
            return self._cache[path]

        try:
            value = self._resolve_path(path)
            if use_cache:
                self._cache[path] = value
            return value
        except (KeyError, IndexError):
            # A KeyError or IndexError during resolution means the path is invalid
            return default

    def set(self, path: str, value, create_scope: bool = True):
        """Set variable with dot notation path support for dicts and lists."""
        # Invalidate cache for this path
        if path in self._cache:
            del self._cache[path]

        parts = path.split('.')

        if len(parts) == 1:
            # Simple key in world_model
            self.world_model[path] = value
            return

        scope_name = parts[0]
        if scope_name not in self.scopes:
            if create_scope:
                self.scopes[scope_name] = {}
            else:
                raise KeyError(f"Scope '{scope_name}' not found")

        current = self.scopes[scope_name]

        # Iterate to the second-to-last part to get the container
        for i, part in enumerate(parts[1:-1]):
            next_part = parts[i + 2]  # Look ahead to the next part in the path

            # Determine if the current part is a dictionary key or a list index
            try:
                # Try to treat it as a list index
                key = int(part)
                if not isinstance(current, list):
                    # If current is not a list, we can't use an integer index
                    raise TypeError(f"Attempted to use integer index '{key}' on non-list for path '{path}'")

                # Ensure list is long enough
                while len(current) <= key:
                    current.append(None)  # Pad with None

                # If the next level doesn't exist, create it based on the next part
                if current[key] is None:
                    current[key] = [] if next_part.isdigit() else {}

                current = current[key]

            except ValueError:
                # It's a dictionary key
                key = part
                if not isinstance(current, dict):
                    raise TypeError(f"Attempted to use string key '{key}' on non-dict for path '{path}'")

                if key not in current:
                    # Create the next level: a list if the next part is a number, else a dict
                    current[key] = [] if next_part.isdigit() else {}

                current = current[key]

        # Handle the final part (the actual assignment)
        last_part = parts[-1]

        if isinstance(current, list):
            try:
                key = int(last_part)
                while len(current) <= key:
                    current.append(None)
                current[key] = value
            except ValueError:
                current.append(value)
        elif isinstance(current, dict):
            current[last_part] = value
        elif scope_name == 'tasks' and hasattr(current, 'task_identification_attr'):# from tasks like Tooltask ... model dump and acces
            dict_data = asdict(current)
            dict_data[last_part] = value
            current = dict_data
            # update self.scopes['tasks'] with the updated task
            self.scopes['tasks'][parts[1]][last_part] = current
        else:
            raise TypeError(f"Final container is not a list or dictionary for path '{path}' its a {type(current)}")

        self._cache.clear()

    def format_text(self, text: str, context: dict = None) -> str:
        """Enhanced text formatting with multiple syntaxes"""
        if not text or not isinstance(text, str):
            return str(text) if text is not None else ""

        # Temporary context overlay
        if context:
            original_scopes = self.scopes.copy()
            self.scopes['context'] = context

        try:
            # Handle {{ variable }} syntax
            formatted = self._format_double_braces(text)

            # Handle {variable} syntax
            formatted = self._format_single_braces(formatted)

            # Handle $variable syntax
            formatted = self._format_dollar_syntax(formatted)

            return formatted

        finally:
            if context:
                self.scopes = original_scopes

    def _format_double_braces(self, text: str) -> str:
        """Handle {{ variable.path }} syntax with improved debugging"""
        import re

        def replace_var(match):
            var_path = match.group(1).strip()
            value = self.get(var_path)

            if value is None:
                # IMPROVED: Log missing variables for debugging
                available_vars = list(self.get_available_variables().keys())
                wprint(f"Variable '{var_path}' not found. Available: {available_vars[:10]}")
                return match.group(0)  # Keep original if not found

            return self._value_to_string(value)

        return re.sub(r'\{\{\s*([^}]+)\s*\}\}', replace_var, text)

    def _format_single_braces(self, text: str) -> str:
        """Handle {variable.path} syntax, including with spaces like { variable.path }."""
        import re

        def replace_var(match):
            # Extrahiert den Variablennamen und entfernt führende/nachfolgende Leerzeichen
            var_path = match.group(1).strip()

            # Ruft den Wert über die get-Methode ab, die die Punktnotation bereits verarbeitet
            value = self.get(var_path)

            # Gibt den konvertierten Wert oder das Original-Tag zurück, wenn der Wert nicht gefunden wurde
            return self._value_to_string(value) if value is not None else match.group(0)

        # Dieser Regex findet {beliebiger.inhalt} und erlaubt Leerzeichen um den Inhalt
        # Er schließt verschachtelte oder leere Klammern wie {} oder { {var} } aus.
        return re.sub(r'\{([^{}]+)\}', replace_var, text)

    def _format_dollar_syntax(self, text: str) -> str:
        """Handle $variable syntax"""
        import re

        def replace_var(match):
            var_name = match.group(1)
            value = self.get(var_name)
            return self._value_to_string(value) if value is not None else match.group(0)

        return re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', replace_var, text)

    def _value_to_string(self, value) -> str:
        """Convert value to string representation"""
        if isinstance(value, str):
            return value
        elif isinstance(value, dict | list):
            return json.dumps(value, default=str)
        else:
            return str(value)

    def validate_references(self, text: str) -> dict[str, bool]:
        """Validate all variable references in text"""
        import re

        references = {}

        # Find all {{ }} references
        double_brace_refs = re.findall(r'\{\{\s*([^}]+)\s*\}\}', text)
        for ref in double_brace_refs:
            references["{{"+ref+"}}"] = self.get(ref.strip()) is not None

        # Find all {} references
        single_brace_refs = re.findall(r'\{([^{}\s]+)\}', text)
        for ref in single_brace_refs:
            if '.' not in ref:  # Only simple vars
                references["{"+ref+"}"] = self.get(ref.strip()) is not None

        # Find all $ references
        dollar_refs = re.findall(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', text)
        for ref in dollar_refs:
            references[f"${ref}"] = self.get(ref) is not None

        return references

    def get_scope_info(self) -> dict[str, Any]:
        """Get information about all available scopes"""
        info = {}
        for scope_name, scope_data in self.scopes.items():
            if isinstance(scope_data, dict):
                info[scope_name] = {
                    'type': 'dict',
                    'keys': len(scope_data),
                    'sample_keys': list(scope_data.keys())[:5]
                }
            else:
                info[scope_name] = {
                    'type': type(scope_data).__name__,
                    'value': str(scope_data)[:100]
                }
        return info

    def _validate_task_references(self, task: Task) -> dict[str, Any]:
        """Validate all variable references in a task"""
        validation_results = {
            'valid': True,
            'errors': [],
            'warnings': []
        }

        # Check different task types
        if isinstance(task, LLMTask):
            if task.prompt_template:
                refs = self.validate_references(task.prompt_template)
                for ref, is_valid in refs.items():
                    if not is_valid:
                        validation_results['errors'].append(f"Invalid reference in prompt: {ref}")
                        validation_results['valid'] = False

        elif isinstance(task, ToolTask):
            for key, value in task.arguments.items():
                if isinstance(value, str):
                    refs = self.validate_references(value)
                    for ref, is_valid in refs.items():
                        if not is_valid:
                            validation_results['warnings'].append(f"Invalid reference in {key}: {ref}")

        return validation_results

    def get_variable_suggestions(self, query: str) -> list[str]:
        """Get variable suggestions based on query content"""

        query_lower = query.lower()
        suggestions = []

        # Check all variables for relevance
        for scope in self.scopes.values():
            for name, var_def in scope.items():
                if name in ["system_context", "task_executor_instance",
                            "index", "tool_capabilities", "use_fast_response", "task_planner_instance"]:
                    continue
                # Name similarity
                if any(word in name.lower() for word in query_lower.split()):
                    suggestions.append(name)
                    continue

                # Description similarity
                if var_def and any(word in str(var_def).lower() for word in query_lower.split()):
                    suggestions.append(name)
                    continue


        return list(set(suggestions))[:10]

    def _document_structure(self, data: Any, path_prefix: str, docs: dict[str, dict]):
        """A recursive helper to document nested dictionaries and lists."""
        if isinstance(data, dict):
            for key, value in data.items():
                # Construct the full path for the current item
                current_path = f"{path_prefix}.{key}" if path_prefix else key

                # Generate a preview for the value
                if isinstance(value, str):
                    preview = value[:70] + "..." if len(value) > 70 else value
                elif isinstance(value, dict):
                    preview = f"Object with keys: {list(value.keys())[:3]}" + ("..." if len(value.keys()) > 3 else "")
                elif isinstance(value, list):
                    preview = f"List with {len(value)} items"
                else:
                    preview = str(value)

                # Store the documentation for the current path
                docs[current_path] = {
                    'preview': preview,
                    'type': type(value).__name__
                }

                # Recurse into nested structures
                if isinstance(value, dict | list):
                    self._document_structure(value, current_path, docs)

        elif isinstance(data, list):
            for i, item in enumerate(data):
                # Construct the full path for the list item
                current_path = f"{path_prefix}.{i}"

                # Generate a preview for the item
                if isinstance(item, str):
                    preview = item[:70] + "..." if len(item) > 70 else item
                elif isinstance(item, dict):
                    preview = f"Object with keys: {list(item.keys())[:3]}" + ("..." if len(item.keys()) > 3 else "")
                elif isinstance(item, list):
                    preview = f"List with {len(item)} items"
                else:
                    preview = str(item)

                docs[current_path] = {
                    'preview': preview,
                    'type': type(item).__name__
                }

                # Recurse into nested structures
                if isinstance(item, dict | list):
                    self._document_structure(item, current_path, docs)

    def get_available_variables(self) -> dict[str, dict]:
        """
        Recursively documents all available variables from world_model and scopes
        to provide a comprehensive overview for an LLM.
        """
        all_vars_docs = {}

        # 1. Document the world_model (top-level variables)
        # self._document_structure(self.world_model, "", all_vars_docs)

        # 2. Document each scope
        for scope_name, scope_data in self.scopes.items():
            # Add documentation for the scope root itself
            if scope_name == "shared":
                continue
            if isinstance(scope_data, dict):
                pass
            elif isinstance(scope_data, list):
                pass
            elif isinstance(scope_data, str | int):
                pass
            else:
                continue

            all_vars_docs[scope_name] = scope_data

            # Recurse into the scope's data
            # self._document_structure(scope_data, scope_name, all_vars_docs)

        return all_vars_docs

    def get_llm_variable_context(self) -> str:
        """
        Generates a detailed variable context formatted for LLM consumption,
        explaining structure, access patterns, and listing all available variables.
        """
        context_parts = [
            "## Variable System Reference",
            "You can access a state management system to retrieve data using dot notation.",
            "Syntax: `{{ path.to.variable }}` or `$path.to.variable`.",
            "",
            "### How to Access Data",
            "The system contains nested objects (dictionaries) and lists (arrays).",
            "",
            "**1. Object (Dictionary) Access (Primary Usage):**",
            "Use a dot (`.`) to access values inside an object. This is the most common way to get data.",
            "Example: If a `user` object exists with a `profile`, you can get the name with `{{ user.profile.name }}`.",
            "",
            "**2. List (Array) Access:**",
            "If a variable is a list, use a dot (`.`) followed by a zero-based number (index) to access a specific item.",
            "Example: To get the first email from a user's email list, use `{{ user.emails.0 }}`.",
            "You can chain these access methods: `{{ user.emails.0.address }}`.",
            "",
            "### Available Variables",
            "Below is a list of all currently available variable paths, their type, and a preview of their content. (Note: Previews may be truncated).",
        ]

        variables = self.get_available_variables()
        if not variables:
            context_parts.append("- No variables are currently set.")
            return "\n".join(context_parts)

        if "shared" in variables:
            variables["shared"] = {'preview': "Shared state variables", 'type': "dict"}

        # yaml dump preview
        context_parts.append("```yaml")
        context_parts.append(yaml.dump(variables, default_flow_style=False, sort_keys=False))
        context_parts.append("```")

        # Add any final complex examples or notes
        context_parts.extend([
            "",
            "**Note on Task Results:**",
            "All task results are stored in the `results` scope. To access the data from a task, append `.data`.",
            "Example: `{{ results.'task-id-123'.data }}`"
        ])

        return "\n".join(context_parts)
format_text(text, context=None)

Enhanced text formatting with multiple syntaxes

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7358
7359
7360
7361
7362
7363
7364
7365
7366
7367
7368
7369
7370
7371
7372
7373
7374
7375
7376
7377
7378
7379
7380
7381
7382
def format_text(self, text: str, context: dict = None) -> str:
    """Enhanced text formatting with multiple syntaxes"""
    if not text or not isinstance(text, str):
        return str(text) if text is not None else ""

    # Temporary context overlay
    if context:
        original_scopes = self.scopes.copy()
        self.scopes['context'] = context

    try:
        # Handle {{ variable }} syntax
        formatted = self._format_double_braces(text)

        # Handle {variable} syntax
        formatted = self._format_single_braces(formatted)

        # Handle $variable syntax
        formatted = self._format_dollar_syntax(formatted)

        return formatted

    finally:
        if context:
            self.scopes = original_scopes
get(path, default=None, use_cache=True)

Get variable with dot notation path support for dicts and lists.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7264
7265
7266
7267
7268
7269
7270
7271
7272
7273
7274
7275
7276
def get(self, path: str, default=None, use_cache: bool = True):
    """Get variable with dot notation path support for dicts and lists."""
    if use_cache and path in self._cache:
        return self._cache[path]

    try:
        value = self._resolve_path(path)
        if use_cache:
            self._cache[path] = value
        return value
    except (KeyError, IndexError):
        # A KeyError or IndexError during resolution means the path is invalid
        return default
get_available_variables()

Recursively documents all available variables from world_model and scopes to provide a comprehensive overview for an LLM.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594
7595
7596
7597
7598
7599
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611
7612
7613
def get_available_variables(self) -> dict[str, dict]:
    """
    Recursively documents all available variables from world_model and scopes
    to provide a comprehensive overview for an LLM.
    """
    all_vars_docs = {}

    # 1. Document the world_model (top-level variables)
    # self._document_structure(self.world_model, "", all_vars_docs)

    # 2. Document each scope
    for scope_name, scope_data in self.scopes.items():
        # Add documentation for the scope root itself
        if scope_name == "shared":
            continue
        if isinstance(scope_data, dict):
            pass
        elif isinstance(scope_data, list):
            pass
        elif isinstance(scope_data, str | int):
            pass
        else:
            continue

        all_vars_docs[scope_name] = scope_data

        # Recurse into the scope's data
        # self._document_structure(scope_data, scope_name, all_vars_docs)

    return all_vars_docs
get_llm_variable_context()

Generates a detailed variable context formatted for LLM consumption, explaining structure, access patterns, and listing all available variables.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7615
7616
7617
7618
7619
7620
7621
7622
7623
7624
7625
7626
7627
7628
7629
7630
7631
7632
7633
7634
7635
7636
7637
7638
7639
7640
7641
7642
7643
7644
7645
7646
7647
7648
7649
7650
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661
7662
def get_llm_variable_context(self) -> str:
    """
    Generates a detailed variable context formatted for LLM consumption,
    explaining structure, access patterns, and listing all available variables.
    """
    context_parts = [
        "## Variable System Reference",
        "You can access a state management system to retrieve data using dot notation.",
        "Syntax: `{{ path.to.variable }}` or `$path.to.variable`.",
        "",
        "### How to Access Data",
        "The system contains nested objects (dictionaries) and lists (arrays).",
        "",
        "**1. Object (Dictionary) Access (Primary Usage):**",
        "Use a dot (`.`) to access values inside an object. This is the most common way to get data.",
        "Example: If a `user` object exists with a `profile`, you can get the name with `{{ user.profile.name }}`.",
        "",
        "**2. List (Array) Access:**",
        "If a variable is a list, use a dot (`.`) followed by a zero-based number (index) to access a specific item.",
        "Example: To get the first email from a user's email list, use `{{ user.emails.0 }}`.",
        "You can chain these access methods: `{{ user.emails.0.address }}`.",
        "",
        "### Available Variables",
        "Below is a list of all currently available variable paths, their type, and a preview of their content. (Note: Previews may be truncated).",
    ]

    variables = self.get_available_variables()
    if not variables:
        context_parts.append("- No variables are currently set.")
        return "\n".join(context_parts)

    if "shared" in variables:
        variables["shared"] = {'preview': "Shared state variables", 'type': "dict"}

    # yaml dump preview
    context_parts.append("```yaml")
    context_parts.append(yaml.dump(variables, default_flow_style=False, sort_keys=False))
    context_parts.append("```")

    # Add any final complex examples or notes
    context_parts.extend([
        "",
        "**Note on Task Results:**",
        "All task results are stored in the `results` scope. To access the data from a task, append `.data`.",
        "Example: `{{ results.'task-id-123'.data }}`"
    ])

    return "\n".join(context_parts)
get_scope_info()

Get information about all available scopes

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7464
7465
7466
7467
7468
7469
7470
7471
7472
7473
7474
7475
7476
7477
7478
7479
def get_scope_info(self) -> dict[str, Any]:
    """Get information about all available scopes"""
    info = {}
    for scope_name, scope_data in self.scopes.items():
        if isinstance(scope_data, dict):
            info[scope_name] = {
                'type': 'dict',
                'keys': len(scope_data),
                'sample_keys': list(scope_data.keys())[:5]
            }
        else:
            info[scope_name] = {
                'type': type(scope_data).__name__,
                'value': str(scope_data)[:100]
            }
    return info
get_variable_suggestions(query)

Get variable suggestions based on query content

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7508
7509
7510
7511
7512
7513
7514
7515
7516
7517
7518
7519
7520
7521
7522
7523
7524
7525
7526
7527
7528
7529
7530
7531
def get_variable_suggestions(self, query: str) -> list[str]:
    """Get variable suggestions based on query content"""

    query_lower = query.lower()
    suggestions = []

    # Check all variables for relevance
    for scope in self.scopes.values():
        for name, var_def in scope.items():
            if name in ["system_context", "task_executor_instance",
                        "index", "tool_capabilities", "use_fast_response", "task_planner_instance"]:
                continue
            # Name similarity
            if any(word in name.lower() for word in query_lower.split()):
                suggestions.append(name)
                continue

            # Description similarity
            if var_def and any(word in str(var_def).lower() for word in query_lower.split()):
                suggestions.append(name)
                continue


    return list(set(suggestions))[:10]
register_scope(name, data)

Register a new variable scope

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7210
7211
7212
7213
def register_scope(self, name: str, data: dict):
    """Register a new variable scope"""
    self.scopes[name] = data
    self._cache.clear()
set(path, value, create_scope=True)

Set variable with dot notation path support for dicts and lists.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7278
7279
7280
7281
7282
7283
7284
7285
7286
7287
7288
7289
7290
7291
7292
7293
7294
7295
7296
7297
7298
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309
7310
7311
7312
7313
7314
7315
7316
7317
7318
7319
7320
7321
7322
7323
7324
7325
7326
7327
7328
7329
7330
7331
7332
7333
7334
7335
7336
7337
7338
7339
7340
7341
7342
7343
7344
7345
7346
7347
7348
7349
7350
7351
7352
7353
7354
7355
7356
def set(self, path: str, value, create_scope: bool = True):
    """Set variable with dot notation path support for dicts and lists."""
    # Invalidate cache for this path
    if path in self._cache:
        del self._cache[path]

    parts = path.split('.')

    if len(parts) == 1:
        # Simple key in world_model
        self.world_model[path] = value
        return

    scope_name = parts[0]
    if scope_name not in self.scopes:
        if create_scope:
            self.scopes[scope_name] = {}
        else:
            raise KeyError(f"Scope '{scope_name}' not found")

    current = self.scopes[scope_name]

    # Iterate to the second-to-last part to get the container
    for i, part in enumerate(parts[1:-1]):
        next_part = parts[i + 2]  # Look ahead to the next part in the path

        # Determine if the current part is a dictionary key or a list index
        try:
            # Try to treat it as a list index
            key = int(part)
            if not isinstance(current, list):
                # If current is not a list, we can't use an integer index
                raise TypeError(f"Attempted to use integer index '{key}' on non-list for path '{path}'")

            # Ensure list is long enough
            while len(current) <= key:
                current.append(None)  # Pad with None

            # If the next level doesn't exist, create it based on the next part
            if current[key] is None:
                current[key] = [] if next_part.isdigit() else {}

            current = current[key]

        except ValueError:
            # It's a dictionary key
            key = part
            if not isinstance(current, dict):
                raise TypeError(f"Attempted to use string key '{key}' on non-dict for path '{path}'")

            if key not in current:
                # Create the next level: a list if the next part is a number, else a dict
                current[key] = [] if next_part.isdigit() else {}

            current = current[key]

    # Handle the final part (the actual assignment)
    last_part = parts[-1]

    if isinstance(current, list):
        try:
            key = int(last_part)
            while len(current) <= key:
                current.append(None)
            current[key] = value
        except ValueError:
            current.append(value)
    elif isinstance(current, dict):
        current[last_part] = value
    elif scope_name == 'tasks' and hasattr(current, 'task_identification_attr'):# from tasks like Tooltask ... model dump and acces
        dict_data = asdict(current)
        dict_data[last_part] = value
        current = dict_data
        # update self.scopes['tasks'] with the updated task
        self.scopes['tasks'][parts[1]][last_part] = current
    else:
        raise TypeError(f"Final container is not a list or dictionary for path '{path}' its a {type(current)}")

    self._cache.clear()
set_results_store(results_store)

Set the results store for task result references

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7215
7216
7217
7218
def set_results_store(self, results_store: dict):
    """Set the results store for task result references"""
    self.scopes['results'] = results_store
    self._cache.clear()
set_tasks_store(tasks_store)

Set tasks store for task metadata access

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7220
7221
7222
7223
def set_tasks_store(self, tasks_store: dict):
    """Set tasks store for task metadata access"""
    self.scopes['tasks'] = tasks_store
    self._cache.clear()
validate_references(text)

Validate all variable references in text

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
7440
7441
7442
7443
7444
7445
7446
7447
7448
7449
7450
7451
7452
7453
7454
7455
7456
7457
7458
7459
7460
7461
7462
def validate_references(self, text: str) -> dict[str, bool]:
    """Validate all variable references in text"""
    import re

    references = {}

    # Find all {{ }} references
    double_brace_refs = re.findall(r'\{\{\s*([^}]+)\s*\}\}', text)
    for ref in double_brace_refs:
        references["{{"+ref+"}}"] = self.get(ref.strip()) is not None

    # Find all {} references
    single_brace_refs = re.findall(r'\{([^{}\s]+)\}', text)
    for ref in single_brace_refs:
        if '.' not in ref:  # Only simple vars
            references["{"+ref+"}"] = self.get(ref.strip()) is not None

    # Find all $ references
    dollar_refs = re.findall(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', text)
    for ref in dollar_refs:
        references[f"${ref}"] = self.get(ref) is not None

    return references
auto_unescape(args)

Automatically unescape all strings in nested data structure.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
12030
12031
12032
def auto_unescape(args: Any) -> Any:
    """Automatically unescape all strings in nested data structure."""
    return process_nested(args)
create_task(task_type, **kwargs)

Factory für Task-Erstellung mit korrektem Typ

Source code in toolboxv2/mods/isaa/base/Agent/types.py
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
def create_task(task_type: str, **kwargs) -> Task:
    """Factory für Task-Erstellung mit korrektem Typ"""
    task_classes = {
        "llm_call": LLMTask,
        "tool_call": ToolTask,
        "decision": DecisionTask,
        "generic": Task,
        "LLMTask": LLMTask,
        "ToolTask": ToolTask,
        "DecisionTask": DecisionTask,
        "Task": Task,
    }

    task_class = task_classes.get(task_type, Task)

    # Standard-Felder setzen
    if "id" not in kwargs:
        kwargs["id"] = str(uuid.uuid4())
    if "type" not in kwargs:
        kwargs["type"] = task_type
    if "critical" not in kwargs:
        kwargs["critical"] = task_type in ["llm_call", "decision"]

    # Ensure metadata is initialized
    if "metadata" not in kwargs:
        kwargs["metadata"] = {}

    # Create task and ensure post_init is called
    task = task_class(**kwargs)

    # Double-check metadata initialization
    if not hasattr(task, 'metadata') or task.metadata is None:
        task.metadata = {}

    return task
get_args_schema(func)

Generate a string representation of a function's arguments and annotations. Keeps args and *kwargs indicators and handles modern Python type hints.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11809
11810
11811
11812
11813
11814
11815
11816
11817
11818
11819
11820
11821
11822
11823
11824
11825
11826
11827
11828
11829
11830
11831
11832
11833
11834
def get_args_schema(func: Callable) -> str:
    """
    Generate a string representation of a function's arguments and annotations.
    Keeps *args and **kwargs indicators and handles modern Python type hints.
    """
    sig = inspect.signature(func)
    parts = []

    for name, param in sig.parameters.items():
        ann = ""
        if param.annotation is not inspect._empty:
            ann = f": {_annotation_to_str(param.annotation)}"

        default = ""
        if param.default is not inspect._empty:
            default = f" = {repr(param.default)}"

        prefix = ""
        if param.kind == inspect.Parameter.VAR_POSITIONAL:
            prefix = "*"
        elif param.kind == inspect.Parameter.VAR_KEYWORD:
            prefix = "**"

        parts.append(f"{prefix}{name}{ann}{default}")

    return f"({', '.join(parts)})"
get_progress_summary(self)

Get comprehensive progress summary from the agent

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11797
11798
11799
11800
11801
def get_progress_summary(self) -> dict[str, Any]:
    """Get comprehensive progress summary from the agent"""
    if hasattr(self, 'progress_tracker'):
        return self.progress_tracker.get_summary()
    return {"error": "No progress tracker available"}
needs_unescaping(text)

Detect if string likely needs unescaping.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
12007
12008
12009
def needs_unescaping(text: str) -> bool:
    """Detect if string likely needs unescaping."""
    return bool(re.search(r'\\[ntr"\'\\]', text)) or len(text) > 50
process_nested(data, max_depth=20)

Recursively process nested structures, unescaping strings that need it.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
12012
12013
12014
12015
12016
12017
12018
12019
12020
12021
12022
12023
12024
12025
12026
12027
def process_nested(data: Any, max_depth: int = 20) -> Any:
    """Recursively process nested structures, unescaping strings that need it."""
    if max_depth <= 0:
        return data

    if isinstance(data, dict):
        return {k: process_nested(v, max_depth - 1) for k, v in data.items()}

    elif isinstance(data, list | tuple):
        processed = [process_nested(item, max_depth - 1) for item in data]
        return type(data)(processed)

    elif isinstance(data, str) and needs_unescaping(data):
        return unescape_string(data)

    return data
unescape_string(text)

Universal string unescaping for any programming language.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
11986
11987
11988
11989
11990
11991
11992
11993
11994
11995
11996
11997
11998
11999
12000
12001
12002
12003
12004
def unescape_string(text: str) -> str:
    """Universal string unescaping for any programming language."""
    if not isinstance(text, str) or len(text) < 2:
        return text

    # Remove outer quotes if wrapped
    if (text.startswith('"') and text.endswith('"')) or (text.startswith("'") and text.endswith("'")):
        text = text[1:-1]

    # Universal escape sequences
    escapes = {
        '\\n': '\n', '\\t': '\t', '\\r': '\r',
        '\\"': '"', "\\'": "'", '\\\\': '\\'
    }

    for escaped, unescaped in escapes.items():
        text = text.replace(escaped, unescaped)

    return text
with_progress_tracking(cls)

Ein Klassendekorator, der die Methoden run_async, prep_async, exec_async, und exec_fallback_async automatisch mit umfassendem Progress-Tracking umwickelt.

Source code in toolboxv2/mods/isaa/base/Agent/agent.py
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
def with_progress_tracking(cls):
    """
    Ein Klassendekorator, der die Methoden run_async, prep_async, exec_async,
    und exec_fallback_async automatisch mit umfassendem Progress-Tracking umwickelt.
    """

    # --- Wrapper für run_async ---
    original_run = getattr(cls, 'run_async', None)
    if original_run:
        @functools.wraps(original_run)
        async def wrapped_run_async(self, shared):
            progress_tracker = shared.get("progress_tracker")
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_run(self, shared)

            timer_key = f"{node_name}_total"
            progress_tracker.start_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_enter",
                timestamp=time.time(),
                node_name=node_name,
                session_id=shared.get("session_id"),
                task_id=shared.get("current_task_id"),
                plan_id=shared.get("current_plan", TaskPlan(id="none", name="none", description="none")).id if shared.get("current_plan") else None,
                status=NodeStatus.RUNNING,
                success=None
            ))

            try:
                # Hier wird die ursprüngliche Methode aufgerufen
                result = await original_run(self, shared)

                total_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_exit",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.COMPLETED,
                    success=True,
                    node_duration=total_duration,
                    routing_decision=result,
                    session_id=shared.get("session_id"),
                    task_id=shared.get("current_task_id"),
                    metadata={"success": True}
                ))

                return result
            except Exception as e:
                total_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="error",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.FAILED,
                    success=False,
                    node_duration=total_duration,
                    session_id=shared.get("session_id"),
                    metadata={"error": str(e), "error_type": type(e).__name__}
                ))
                raise

        cls.run_async = wrapped_run_async

    # --- Wrapper für prep_async ---
    original_prep = getattr(cls, 'prep_async', None)
    if original_prep:
        @functools.wraps(original_prep)
        async def wrapped_prep_async(self, shared):
            progress_tracker = shared.get("progress_tracker")
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_prep(self, shared)
            timer_key = f"{node_name}_total_p"
            progress_tracker.start_timer(timer_key)
            timer_key = f"{node_name}_prep"
            progress_tracker.start_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.STARTING,
                node_phase="prep",
                session_id=shared.get("session_id")
            ))

            try:
                result = await original_prep(self, shared)

                prep_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_phase",
                    timestamp=time.time(),
                    status=NodeStatus.RUNNING,
                    success=True,
                    node_name=node_name,
                    node_phase="prep_complete",
                    node_duration=prep_duration,
                    session_id=shared.get("session_id")
                ))
                return result
            except Exception as e:
                progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="error",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.FAILED,
                    success=False,
                    metadata={"error": str(e), "error_type": type(e).__name__},
                    node_phase="prep_failed"
                ))
                raise


        cls.prep_async = wrapped_prep_async

    # --- Wrapper für exec_async ---
    original_exec = getattr(cls, 'exec_async', None)
    if original_exec:
        @functools.wraps(original_exec)
        async def wrapped_exec_async(self, prep_res):
            progress_tracker = prep_res.get("progress_tracker") if isinstance(prep_res, dict) else None
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_exec(self, prep_res)

            timer_key = f"{node_name}_exec"
            progress_tracker.start_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.RUNNING,
                node_phase="exec",
                session_id=prep_res.get("session_id") if isinstance(prep_res, dict) else None
            ))

            # In exec gibt es normalerweise keine Fehlerbehandlung, da diese von run_async übernommen wird
            result = await original_exec(self, prep_res)

            exec_duration = progress_tracker.end_timer(timer_key)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.RUNNING,
                success=True,
                node_phase="exec_complete",
                node_duration=exec_duration,
                session_id=prep_res.get("session_id") if isinstance(prep_res, dict) else None
            ))
            return result

        cls.exec_async = wrapped_exec_async

    # --- Wrapper für post_async ---
    original_post = getattr(cls, 'post_async', None)
    if original_post:
        @functools.wraps(original_post)
        async def wrapped_post_async(self, shared, prep_res, exec_res):
            if isinstance(exec_res, str):
                print("exec_res is string:", exec_res)
            progress_tracker = shared.get("progress_tracker")
            node_name = self.__class__.__name__

            if not progress_tracker:
                return await original_post(self, shared, prep_res, exec_res)

            timer_key_post = f"{node_name}_post"
            progress_tracker.start_timer(timer_key_post)
            await progress_tracker.emit_event(ProgressEvent(
                event_type="node_phase",
                timestamp=time.time(),
                node_name=node_name,
                status=NodeStatus.COMPLETING,  # Neue Phase "completing"
                node_phase="post",
                session_id=shared.get("session_id")
            ))

            try:
                # Die eigentliche post_async Methode aufrufen
                result = await original_post(self, shared, prep_res, exec_res)

                post_duration = progress_tracker.end_timer(timer_key_post)
                total_duration = progress_tracker.end_timer(f"{node_name}_total_p")  # Gesamtdauer stoppen

                # Sende das entscheidende "node_exit" Event nach erfolgreicher post-Phase
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_exit",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.COMPLETED,
                    success=True,
                    node_duration=total_duration,
                    routing_decision=result,
                    session_id=shared.get("session_id"),
                    task_id=shared.get("current_task_id"),
                    metadata={
                        "success": True,
                        "post_duration": post_duration
                    }
                ))

                return result
            except Exception as e:
                # Fehler in der post-Phase

                post_duration = progress_tracker.end_timer(timer_key_post)
                total_duration = progress_tracker.end_timer(f"{node_name}_total")
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="error",
                    timestamp=time.time(),
                    node_name=node_name,
                    status=NodeStatus.FAILED,
                    success=False,
                    node_duration=total_duration,
                    metadata={"error": str(e), "error_type": type(e).__name__, "phase": "post"},
                    node_phase="post_failed"
                ))
                raise

        cls.post_async = wrapped_post_async

    # --- Wrapper für exec_fallback_async ---
    original_fallback = getattr(cls, 'exec_fallback_async', None)
    if original_fallback:
        @functools.wraps(original_fallback)
        async def wrapped_fallback_async(self, prep_res, exc):
            progress_tracker = prep_res.get("progress_tracker") if isinstance(prep_res, dict) else None
            node_name = self.__class__.__name__

            if progress_tracker:
                timer_key = f"{node_name}_exec"
                exec_duration = progress_tracker.end_timer(timer_key)
                await progress_tracker.emit_event(ProgressEvent(
                    event_type="node_phase",
                    timestamp=time.time(),
                    node_name=node_name,
                    node_phase="exec_fallback",
                    node_duration=exec_duration,
                    status=NodeStatus.FAILED,
                    success=False,
                    session_id=prep_res.get("session_id") if isinstance(prep_res, dict) else None,
                    metadata={"error": str(exc), "error_type": type(exc).__name__},
                ))

            return await original_fallback(self, prep_res, exc)

        cls.exec_fallback_async = wrapped_fallback_async

    return cls
builder
A2AConfig

Bases: BaseModel

A2A server configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
100
101
102
103
104
105
106
107
108
109
110
class A2AConfig(BaseModel):
    """A2A server configuration"""
    model_config = ConfigDict(arbitrary_types_allowed=True)

    enabled: bool = False
    host: str = "0.0.0.0"
    port: int = 5000
    agent_name: Optional[str] = None
    agent_description: Optional[str] = None
    agent_version: str = "1.0.0"
    expose_tools_as_skills: bool = True
AgentConfig

Bases: BaseModel

Complete agent configuration for loading/saving

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
class AgentConfig(BaseModel):
    """Complete agent configuration for loading/saving"""
    model_config = ConfigDict(arbitrary_types_allowed=True)

    # Basic settings
    name: str = "ProductionAgent"
    description: str = "Production-ready PocketFlow agent"
    version: str = "2.0.0"

    # LLM settings
    fast_llm_model: str = "openrouter/anthropic/claude-3-haiku"
    complex_llm_model: str = "openrouter/openai/gpt-4o"
    system_message: str = """You are a production-ready autonomous agent with advanced capabilities including:
- Native MCP tool integration for extensible functionality
- A2A compatibility for agent-to-agent communication
- Dynamic task planning and execution with adaptive reflection
- Advanced context management with session awareness
- Variable system for dynamic content generation
- Checkpoint/resume capabilities for reliability

Always utilize available tools when they can help solve the user's request efficiently."""

    temperature: float = 0.7
    max_tokens_output: int = 2048
    max_tokens_input: int = 32768
    api_key_env_var: str | None = "OPENROUTER_API_KEY"
    use_fast_response: bool = True

    # Features
    mcp: MCPConfig = Field(default_factory=MCPConfig)
    a2a: A2AConfig = Field(default_factory=A2AConfig)
    telemetry: TelemetryConfig = Field(default_factory=TelemetryConfig)
    checkpoint: CheckpointConfig = Field(default_factory=CheckpointConfig)

    # Agent behavior
    max_parallel_tasks: int = 3
    verbose_logging: bool = False

    # Persona and formatting
    active_persona: Optional[str] = None
    persona_profiles: dict[str, dict[str, Any]] = Field(default_factory=dict)
    default_format_config: Optional[dict[str, Any]] = None

    # Custom variables and world model
    custom_variables: dict[str, Any] = Field(default_factory=dict)
    initial_world_model: dict[str, Any] = Field(default_factory=dict)
CheckpointConfig

Bases: BaseModel

Checkpoint configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
123
124
125
126
127
128
129
class CheckpointConfig(BaseModel):
    """Checkpoint configuration"""
    enabled: bool = True
    interval_seconds: int = 300  # 5 minutes
    max_checkpoints: int = 10
    checkpoint_dir: str = "./checkpoints"
    auto_save_on_exit: bool = True
FlowAgentBuilder

Production-ready FlowAgent builder focused on MCP, A2A, and robust deployment

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
class FlowAgentBuilder:
    """Production-ready FlowAgent builder focused on MCP, A2A, and robust deployment"""

    def __init__(self, config: AgentConfig = None, config_path: str = None):
        """Initialize builder with configuration"""

        if config and config_path:
            raise ValueError("Provide either config object or config_path, not both")

        if config_path:
            self.config = self.load_config(config_path)
        elif config:
            self.config = config
        else:
            self.config = AgentConfig()

        # Runtime components
        self._custom_tools: dict[str, tuple[Callable, str]] = {}
        self._mcp_tools: dict[str, dict] = {}
        from toolboxv2.mods.isaa.extras.mcp_session_manager import MCPSessionManager

        self._mcp_session_manager = MCPSessionManager()

        self._budget_manager: BudgetManager = None
        self._tracer_provider: TracerProvider = None
        self._a2a_server: Any = None

        # Set logging level
        if self.config.verbose_logging:
            logging.getLogger().setLevel(logging.DEBUG)

        iprint(f"FlowAgent Builder initialized: {self.config.name}")

    # ===== CONFIGURATION MANAGEMENT =====

    def load_config(self, config_path: str) -> AgentConfig:
        """Load agent configuration from file"""
        path = Path(config_path)
        if not path.exists():
            raise FileNotFoundError(f"Config file not found: {config_path}")

        try:
            with open(path, encoding='utf-8') as f:
                if path.suffix.lower() in ['.yaml', '.yml']:
                    data = yaml.safe_load(f)
                else:
                    data = json.load(f)

            return AgentConfig(**data)

        except Exception as e:
            eprint(f"Failed to load config from {config_path}: {e}")
            raise

    def save_config(self, config_path: str, format: str = 'yaml'):
        """Save current configuration to file"""
        path = Path(config_path)
        path.parent.mkdir(parents=True, exist_ok=True)

        try:
            data = self.config.model_dump()

            with open(path, 'w', encoding='utf-8') as f:
                if format.lower() == 'yaml':
                    yaml.dump(data, f, default_flow_style=False, indent=2)
                else:
                    json.dump(data, f, indent=2)

            iprint(f"Configuration saved to {config_path}")

        except Exception as e:
            eprint(f"Failed to save config to {config_path}: {e}")
            raise

    @classmethod
    def from_config_file(cls, config_path: str) -> 'FlowAgentBuilder':
        """Create builder from configuration file"""
        return cls(config_path=config_path)

    # ===== FLUENT BUILDER API =====

    def with_name(self, name: str) -> 'FlowAgentBuilder':
        """Set agent name"""
        self.config.name = name
        return self

    def with_models(self, fast_model: str, complex_model: str = None) -> 'FlowAgentBuilder':
        """Set LLM models"""
        self.config.fast_llm_model = fast_model
        if complex_model:
            self.config.complex_llm_model = complex_model
        return self

    def with_system_message(self, message: str) -> 'FlowAgentBuilder':
        """Set system message"""
        self.config.system_message = message
        return self

    def with_temperature(self, temp: float) -> 'FlowAgentBuilder':
        """Set temperature"""
        self.config.temperature = temp
        return self

    def with_budget_manager(self, max_cost: float = 10.0) -> 'FlowAgentBuilder':
        """Enable budget management"""
        if LITELLM_AVAILABLE:
            self._budget_manager = BudgetManager("agent")
            iprint(f"Budget manager enabled: ${max_cost}")
        else:
            wprint("LiteLLM not available, budget manager disabled")
        return self

    def verbose(self, enable: bool = True) -> 'FlowAgentBuilder':
        """Enable verbose logging"""
        self.config.verbose_logging = enable
        if enable:
            logging.getLogger().setLevel(logging.DEBUG)
        return self

    # ===== MCP INTEGRATION =====

    def enable_mcp_server(self, host: str = "0.0.0.0", port: int = 8000,
                          server_name: str = None) -> 'FlowAgentBuilder':
        """Enable MCP server"""
        if not MCP_AVAILABLE:
            wprint("MCP not available, cannot enable server")
            return self

        self.config.mcp.enabled = True
        self.config.mcp.host = host
        self.config.mcp.port = port
        self.config.mcp.server_name = server_name or f"{self.config.name}_MCP"

        iprint(f"MCP server enabled: {host}:{port}")
        return self

    async def _load_mcp_server_capabilities(self, server_name: str, server_config: dict[str, Any]):
        """Load all capabilities from MCP server with persistent session"""
        try:
            # Get or create persistent session
            session = await self._mcp_session_manager.get_session(server_name, server_config)
            if not session:
                eprint(f"Failed to create session for MCP server: {server_name}")
                return

            # Extract all capabilities
            capabilities = await self._mcp_session_manager.extract_capabilities(session, server_name)

            # Create tool wrappers
            for tool_name, tool_info in capabilities['tools'].items():
                wrapper_name = f"{server_name}_{tool_name}"
                tool_wrapper = self._create_tool_wrapper(server_name, tool_name, tool_info, session)
                self._mcp_tools[wrapper_name] = {
                    'function': tool_wrapper,
                    'description': tool_info['description'],
                    'type': 'tool',
                    'server': server_name,
                    'original_name': tool_name,
                    'input_schema': tool_info.get('input_schema'),
                    'output_schema': tool_info.get('output_schema')
                }

            # Create resource wrappers
            for resource_uri, resource_info in capabilities['resources'].items():
                wrapper_name = f"{server_name}_resource_{resource_info['name'].replace('/', '_')}"
                resource_wrapper = self._create_resource_wrapper(server_name, resource_uri, resource_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': resource_wrapper,
                    'description': f"Read resource: {resource_info['description']}",
                    'type': 'resource',
                    'server': server_name,
                    'original_uri': resource_uri
                }

            # Create resource template wrappers
            for template_uri, template_info in capabilities['resource_templates'].items():
                wrapper_name = f"{server_name}_template_{template_info['name'].replace('/', '_')}"
                template_wrapper = self._create_resource_template_wrapper(server_name, template_uri, template_info,
                                                                          session)

                self._mcp_tools[wrapper_name] = {
                    'function': template_wrapper,
                    'description': f"Access resource template: {template_info['description']}",
                    'type': 'resource_template',
                    'server': server_name,
                    'original_template': template_uri
                }

            # Create prompt wrappers
            for prompt_name, prompt_info in capabilities['prompts'].items():
                wrapper_name = f"{server_name}_prompt_{prompt_name}"
                prompt_wrapper = self._create_prompt_wrapper(server_name, prompt_name, prompt_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': prompt_wrapper,
                    'description': f"Execute prompt: {prompt_info['description']}",
                    'type': 'prompt',
                    'server': server_name,
                    'original_name': prompt_name,
                    'arguments': prompt_info.get('arguments', [])
                }

            total_capabilities = (len(capabilities['tools']) +
                                  len(capabilities['resources']) +
                                  len(capabilities['resource_templates']) +
                                  len(capabilities['prompts']))

            iprint(f"Created {total_capabilities} capability wrappers for server: {server_name}")

        except Exception as e:
            eprint(f"Failed to load capabilities from MCP server {server_name}: {e}")

    def _create_tool_wrapper(self, server_name: str, tool_name: str, tool_info: dict, session: ClientSession):
        """Create wrapper function for MCP tool with dynamic signature based on schema"""
        import inspect

        # Extract parameter information from input schema
        input_schema = tool_info.get('input_schema', {})
        output_schema = tool_info.get('output_schema', {})

        # Build parameter list
        parameters = []
        required_params = set(input_schema.get('required', []))
        properties = input_schema.get('properties', {})

        # Create parameters with proper types
        for param_name, param_info in properties.items():
            param_type = param_info.get('type', 'string')
            python_type = {
                'string': str,
                'integer': int,
                'number': float,
                'boolean': bool,
                'array': list,
                'object': dict
            }.get(param_type, str)

            # Determine if parameter is required
            if param_name in required_params:
                param = inspect.Parameter(param_name, inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=python_type)
            else:
                # Optional parameters get default None
                param = inspect.Parameter(param_name, inspect.Parameter.POSITIONAL_OR_KEYWORD,
                                          annotation=python_type, default=None)
            parameters.append(param)

        # Determine return type from output schema
        return_type = str  # Default
        if output_schema and 'properties' in output_schema:
            output_props = output_schema['properties']
            if len(output_props) == 1:
                # Single property, return its type directly
                prop_info = list(output_props.values())[0]
                prop_type = prop_info.get('type', 'string')
                return_type = {
                    'string': str,
                    'integer': int,
                    'number': float,
                    'boolean': bool,
                    'array': list,
                    'object': dict
                }.get(prop_type, str)
            else:
                # Multiple properties, return dict
                return_type = dict

        # Create the actual function
        async def tool_wrapper(*args, **kwargs):
            try:
                # Map arguments to schema parameters
                arguments = {}
                param_names = list(properties.keys())

                # Map positional args
                for i, arg in enumerate(args):
                    if i < len(param_names):
                        arguments[param_names[i]] = arg

                # Add keyword arguments, filtering out None for optional params
                for key, value in kwargs.items():
                    if value is not None or key in required_params:
                        arguments[key] = value

                # Validate required parameters
                missing_required = required_params - set(arguments.keys())
                if missing_required:
                    raise ValueError(f"Missing required parameters: {missing_required}")

                # Call the actual MCP tool
                result = await session.call_tool(tool_name, arguments)

                # Handle structured vs unstructured results
                if hasattr(result, 'structuredContent') and result.structuredContent:
                    structured_data = result.structuredContent

                    # If output schema expects single property, extract it
                    if output_schema and 'properties' in output_schema:
                        output_props = output_schema['properties']
                        if len(output_props) == 1:
                            prop_name = list(output_props.keys())[0]
                            if isinstance(structured_data, dict) and prop_name in structured_data:
                                return structured_data[prop_name]

                    return structured_data

                # Fallback to content extraction
                if result.content:
                    content = result.content[0]
                    if hasattr(content, 'text'):
                        return content.text
                    elif hasattr(content, 'data'):
                        return content.data
                    else:
                        return str(content)

                return "No content returned"

            except Exception as e:
                eprint(f"MCP tool {server_name}.{tool_name} failed: {e}")
                raise RuntimeError(f"Error executing {tool_name}: {str(e)}")

        # Set dynamic signature
        signature = inspect.Signature(parameters, return_annotation=return_type)
        tool_wrapper.__signature__ = signature
        tool_wrapper.__name__ = f"{server_name}_{tool_name}"
        tool_wrapper.__doc__ = tool_info.get('description', f"MCP tool: {tool_name}")
        tool_wrapper.__annotations__ = {'return': return_type}

        # Add parameter annotations
        for param in parameters:
            tool_wrapper.__annotations__[param.name] = param.annotation

        return tool_wrapper

    def _create_resource_wrapper(self, server_name: str, resource_uri: str, resource_info: dict,
                                 session: ClientSession):
        """Create wrapper function for MCP resource with proper signature"""
        import inspect

        # Resources typically don't take parameters, return string content
        async def resource_wrapper() -> str:
            """Read MCP resource content"""
            try:
                from pydantic import AnyUrl
                result = await session.read_resource(AnyUrl(resource_uri))

                if result.contents:
                    content = result.contents[0]
                    if hasattr(content, 'text'):
                        return content.text
                    elif hasattr(content, 'data'):
                        # Handle binary data
                        if isinstance(content.data, bytes):
                            return content.data.decode('utf-8', errors='ignore')
                        return str(content.data)
                    else:
                        return str(content)

                return ""

            except Exception as e:
                eprint(f"MCP resource {resource_uri} failed: {e}")
                raise RuntimeError(f"Error reading resource: {str(e)}")

        # Set signature and metadata
        signature = inspect.Signature([], return_annotation=str)
        resource_wrapper.__signature__ = signature
        resource_wrapper.__name__ = f"{server_name}_resource_{resource_info['name'].replace('/', '_').replace(':', '_')}"
        resource_wrapper.__doc__ = f"Read MCP resource: {resource_info.get('description', resource_uri)}"
        resource_wrapper.__annotations__ = {'return': str}

        return resource_wrapper

    def _create_resource_template_wrapper(self, server_name: str, template_uri: str, template_info: dict,
                                          session: ClientSession):
        """Create wrapper function for MCP resource template with dynamic parameters"""
        import inspect
        import re

        # Extract template variables from URI (e.g., {owner}, {repo})
        template_vars = re.findall(r'\{(\w+)\}', template_uri)

        # Create parameters for each template variable
        parameters = []
        for var_name in template_vars:
            param = inspect.Parameter(var_name, inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=str)
            parameters.append(param)

        async def template_wrapper(*args, **kwargs) -> str:
            """Access MCP resource template with parameters"""
            try:
                from pydantic import AnyUrl

                # Map arguments to template variables
                template_args = {}
                for i, arg in enumerate(args):
                    if i < len(template_vars):
                        template_args[template_vars[i]] = arg

                template_args.update(kwargs)

                # Validate all required template variables are provided
                missing_vars = set(template_vars) - set(template_args.keys())
                if missing_vars:
                    raise ValueError(f"Missing required template variables: {missing_vars}")

                # Replace template variables in URI
                actual_uri = template_uri
                for var_name, value in template_args.items():
                    actual_uri = actual_uri.replace(f"{{{var_name}}}", str(value))

                result = await session.read_resource(AnyUrl(actual_uri))

                if result.contents:
                    content = result.contents[0]
                    if hasattr(content, 'text'):
                        return content.text
                    elif hasattr(content, 'data'):
                        if isinstance(content.data, bytes):
                            return content.data.decode('utf-8', errors='ignore')
                        return str(content.data)
                    else:
                        return str(content)

                return ""

            except Exception as e:
                eprint(f"MCP resource template {template_uri} failed: {e}")
                raise RuntimeError(f"Error accessing resource template: {str(e)}")

        # Set dynamic signature
        signature = inspect.Signature(parameters, return_annotation=str)
        template_wrapper.__signature__ = signature
        template_wrapper.__name__ = f"{server_name}_template_{template_info['name'].replace('/', '_').replace(':', '_')}"
        template_wrapper.__doc__ = f"Access MCP resource template: {template_info.get('description', template_uri)}\nTemplate variables: {', '.join(template_vars)}"
        template_wrapper.__annotations__ = {'return': str}

        # Add parameter annotations
        for param in parameters:
            template_wrapper.__annotations__[param.name] = str

        return template_wrapper

    def _create_prompt_wrapper(self, server_name: str, prompt_name: str, prompt_info: dict, session: ClientSession):
        """Create wrapper function for MCP prompt with dynamic parameters"""
        import inspect

        # Extract parameter information from prompt arguments
        prompt_args = prompt_info.get('arguments', [])

        # Create parameters
        parameters = []
        for arg_info in prompt_args:
            arg_name = arg_info['name']
            is_required = arg_info.get('required', False)

            if is_required:
                param = inspect.Parameter(arg_name, inspect.Parameter.POSITIONAL_OR_KEYWORD, annotation=str)
            else:
                param = inspect.Parameter(arg_name, inspect.Parameter.POSITIONAL_OR_KEYWORD,
                                          annotation=str, default=None)
            parameters.append(param)

        async def prompt_wrapper(*args, **kwargs) -> str:
            """Execute MCP prompt with parameters"""
            try:
                # Map arguments
                prompt_arguments = {}
                arg_names = [arg['name'] for arg in prompt_args]

                # Map positional args
                for i, arg in enumerate(args):
                    if i < len(arg_names):
                        prompt_arguments[arg_names[i]] = arg

                # Add keyword arguments, filtering None for optional params
                required_args = {arg['name'] for arg in prompt_args if arg.get('required', False)}
                for key, value in kwargs.items():
                    if value is not None or key in required_args:
                        prompt_arguments[key] = value

                # Validate required parameters
                missing_required = required_args - set(prompt_arguments.keys())
                if missing_required:
                    raise ValueError(f"Missing required prompt arguments: {missing_required}")

                result = await session.get_prompt(prompt_name, prompt_arguments)

                # Extract and combine messages
                messages = []
                for message in result.messages:
                    if hasattr(message.content, 'text'):
                        messages.append(message.content.text)
                    else:
                        messages.append(str(message.content))

                return "\n".join(messages) if messages else ""

            except Exception as e:
                eprint(f"MCP prompt {prompt_name} failed: {e}")
                raise RuntimeError(f"Error executing prompt: {str(e)}")

        # Set dynamic signature
        signature = inspect.Signature(parameters, return_annotation=str)
        prompt_wrapper.__signature__ = signature
        prompt_wrapper.__name__ = f"{server_name}_prompt_{prompt_name}"

        # Build docstring with parameter info
        param_docs = []
        for arg_info in prompt_args:
            required_str = "required" if arg_info.get('required', False) else "optional"
            param_docs.append(
                f"    {arg_info['name']} ({required_str}): {arg_info.get('description', 'No description')}")

        docstring = f"Execute MCP prompt: {prompt_info.get('description', prompt_name)}"
        if param_docs:
            docstring += "\n\nParameters:\n" + "\n".join(param_docs)

        prompt_wrapper.__doc__ = docstring
        prompt_wrapper.__annotations__ = {'return': str}

        # Add parameter annotations
        for param in parameters:
            prompt_wrapper.__annotations__[param.name] = str

        return prompt_wrapper

    def load_mcp_tools_from_config(self, config_path: str | dict) -> 'FlowAgentBuilder':
        """Enhanced MCP config loading with automatic session management and full capability extraction"""
        if not MCP_AVAILABLE:
            wprint("MCP not available, skipping tool loading")
            return self

        if isinstance(config_path, dict):
            mcp_config = config_path
            from toolboxv2 import get_app
            name = self.config.name or "inline_config"
            path = Path(get_app().appdata) / "isaa" / "MCPConfig" / f"{name}.json"
            path.parent.mkdir(parents=True, exist_ok=True)
            path.write_text(json.dumps(mcp_config, indent=2))
            config_path = path
        else:
            config_path = Path(config_path)
            if not config_path.exists():
                raise FileNotFoundError(f"MCP config not found: {config_path}")

            try:
                with open(config_path, encoding='utf-8') as f:
                    if config_path.suffix.lower() in ['.yaml', '.yml']:
                        mcp_config = yaml.safe_load(f)
                    else:
                        mcp_config = json.load(f)

            except Exception as e:
                eprint(f"Failed to load MCP config from {config_path}: {e}")
                raise

        # Store config for async processing
        self._mcp_config_data = mcp_config
        self.config.mcp.config_path = str(config_path)

        # Mark for processing during build
        self._mcp_needs_loading = True

        iprint(f"MCP config loaded from {config_path}, will process during build")

        return self

    async def _process_mcp_config(self):
        """Process MCP configuration with proper task management"""
        if not hasattr(self, '_mcp_config_data') or not self._mcp_config_data:
            return

        mcp_config = self._mcp_config_data

        # Handle standard MCP server configuration with sequential processing to avoid task issues
        if 'mcpServers' in mcp_config:
            servers_to_load = []

            # Validate all servers first
            for server_name, server_config in mcp_config['mcpServers'].items():
                if self._validate_mcp_server_config(server_name, server_config):
                    servers_to_load.append((server_name, server_config))
                else:
                    wprint(f"Skipping invalid MCP server config: {server_name}")

            if servers_to_load:
                iprint(f"Processing {len(servers_to_load)} MCP servers sequentially...")

                # Process servers sequentially to avoid task boundary issues
                successful_loads = 0
                for server_name, server_config in servers_to_load:
                    try:
                        result = await asyncio.wait_for(
                            self._load_single_mcp_server(server_name, server_config),
                            timeout=5.0  # Per-server timeout
                        )

                        if result:
                            successful_loads += 1
                            iprint(f"✓ Successfully loaded MCP server: {server_name}")
                        else:
                            wprint(f"⚠ MCP server {server_name} loaded with issues")

                    except TimeoutError:
                        eprint(f"✗ MCP server {server_name} timed out after 15 seconds")
                    except Exception as e:
                        eprint(f"✗ Failed to load MCP server {server_name}: {e}")

                iprint(
                    f"MCP processing complete: {successful_loads}/{len(servers_to_load)} servers loaded successfully")

        # Handle direct tools configuration (legacy)
        elif 'tools' in mcp_config:
            for tool_config in mcp_config['tools']:
                try:
                    self._load_direct_mcp_tool(tool_config)
                except Exception as e:
                    eprint(f"Failed to load direct MCP tool: {e}")

    async def _load_single_mcp_server(self, server_name: str, server_config: dict[str, Any]) -> bool:
        """Load a single MCP server with timeout and error handling"""
        try:
            iprint(f"🔄 Processing MCP server: {server_name}")

            # Get session with timeout
            session = await self._mcp_session_manager.get_session_with_timeout(server_name, server_config)
            if not session:
                eprint(f"✗ Failed to create session for MCP server: {server_name}")
                return False

            # Extract capabilities with timeout
            capabilities = await self._mcp_session_manager.extract_capabilities_with_timeout(session, server_name)
            if not any(capabilities.values()):
                wprint(f"⚠ No capabilities found for MCP server: {server_name}")
                return False

            # Create wrappers for all capabilities
            await self._create_capability_wrappers(server_name, capabilities, session)

            total_caps = sum(len(caps) for caps in capabilities.values())
            iprint(f"✓ Created {total_caps} capability wrappers for: {server_name}")

            return True

        except Exception as e:
            eprint(f"✗ Error loading MCP server {server_name}: {e}")
            return False

    async def _create_capability_wrappers(self, server_name: str, capabilities: dict, session: ClientSession):
        """Create wrappers for all capabilities with error handling"""

        # Create tool wrappers
        for tool_name, tool_info in capabilities['tools'].items():
            try:
                wrapper_name = f"{server_name}_{tool_name}"
                tool_wrapper = self._create_tool_wrapper(server_name, tool_name, tool_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': tool_wrapper,
                    'description': tool_info['description'],
                    'type': 'tool',
                    'server': server_name,
                    'original_name': tool_name,
                    'input_schema': tool_info.get('input_schema'),
                    'output_schema': tool_info.get('output_schema')
                }
            except Exception as e:
                eprint(f"Failed to create tool wrapper {tool_name}: {e}")

        # Create resource wrappers
        for resource_uri, resource_info in capabilities['resources'].items():
            try:
                safe_name = resource_info['name'].replace('/', '_').replace(':', '_')
                wrapper_name = f"{server_name}_resource_{safe_name}"
                resource_wrapper = self._create_resource_wrapper(server_name, resource_uri, resource_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': resource_wrapper,
                    'description': f"Read resource: {resource_info['description']}",
                    'type': 'resource',
                    'server': server_name,
                    'original_uri': resource_uri
                }
            except Exception as e:
                eprint(f"Failed to create resource wrapper {resource_uri}: {e}")

        # Create resource template wrappers
        for template_uri, template_info in capabilities['resource_templates'].items():
            try:
                safe_name = template_info['name'].replace('/', '_').replace(':', '_')
                wrapper_name = f"{server_name}_template_{safe_name}"
                template_wrapper = self._create_resource_template_wrapper(server_name, template_uri, template_info,
                                                                          session)

                self._mcp_tools[wrapper_name] = {
                    'function': template_wrapper,
                    'description': f"Access resource template: {template_info['description']}",
                    'type': 'resource_template',
                    'server': server_name,
                    'original_template': template_uri
                }
            except Exception as e:
                eprint(f"Failed to create template wrapper {template_uri}: {e}")

        # Create prompt wrappers
        for prompt_name, prompt_info in capabilities['prompts'].items():
            try:
                wrapper_name = f"{server_name}_prompt_{prompt_name}"
                prompt_wrapper = self._create_prompt_wrapper(server_name, prompt_name, prompt_info, session)

                self._mcp_tools[wrapper_name] = {
                    'function': prompt_wrapper,
                    'description': f"Execute prompt: {prompt_info['description']}",
                    'type': 'prompt',
                    'server': server_name,
                    'original_name': prompt_name,
                    'arguments': prompt_info.get('arguments', [])
                }
            except Exception as e:
                eprint(f"Failed to create prompt wrapper {prompt_name}: {e}")

    @staticmethod
    def _validate_mcp_server_config(server_name: str, server_config: dict[str, Any]) -> bool:
        """Validate MCP server configuration"""
        command = server_config.get('command')
        if not command:
            eprint(f"MCP server {server_name} missing 'command' field")
            return False

        # Check if command exists and is executable
        if command in ['npx', 'node', 'python', 'python3', 'docker']:
            # These are common commands, assume they exist
            return True

        if server_config.get('transport') in ['http', 'streamable-http'] and server_config.get('url'):
            return True

        # For other commands, check if they exist
        import shutil
        if not shutil.which(command):
            wprint(f"MCP server {server_name}: command '{command}' not found in PATH")
            # Don't fail completely, just warn - the command might be available at runtime

        args = server_config.get('args', [])
        if not isinstance(args, list):
            eprint(f"MCP server {server_name}: 'args' must be a list")
            return False

        env = server_config.get('env', {})
        if not isinstance(env, dict):
            eprint(f"MCP server {server_name}: 'env' must be a dictionary")
            return False

        iprint(f"Validated MCP server config: {server_name}")
        return True

    def _load_direct_mcp_tool(self, tool_config: dict[str, Any]):
        """Load tool from direct configuration"""
        name = tool_config.get('name')
        description = tool_config.get('description', '')
        function_code = tool_config.get('function_code')

        if not name or not function_code:
            wprint(f"Incomplete tool config: {tool_config}")
            return

        # Create function from code
        try:
            namespace = {"__builtins__": __builtins__}
            exec(function_code, namespace)

            # Find the function
            func = None
            for obj in namespace.values():
                if callable(obj) and not getattr(obj, '__name__', '').startswith('_'):
                    func = obj
                    break

            if func:
                self._mcp_tools[name] = {
                    'function': func,
                    'description': description,
                    'source': 'code'
                }
                iprint(f"Loaded MCP tool from code: {name}")

        except Exception as e:
            eprint(f"Failed to load MCP tool {name}: {e}")

    def add_mcp_tool_from_code(self, name: str, code: str, description: str = "") -> 'FlowAgentBuilder':
        """Add MCP tool from code string"""
        tool_config = {
            'name': name,
            'description': description,
            'function_code': code
        }
        self._load_direct_mcp_tool(tool_config)
        return self

    # ===== A2A INTEGRATION =====

    def enable_a2a_server(self, host: str = "0.0.0.0", port: int = 5000,
                          agent_name: str = None, agent_description: str = None) -> 'FlowAgentBuilder':
        """Enable A2A server for agent-to-agent communication"""
        if not A2A_AVAILABLE:
            wprint("A2A not available, cannot enable server")
            return self

        self.config.a2a.enabled = True
        self.config.a2a.host = host
        self.config.a2a.port = port
        self.config.a2a.agent_name = agent_name or self.config.name
        self.config.a2a.agent_description = agent_description or self.config.description

        iprint(f"A2A server enabled: {host}:{port}")
        return self

    # ===== TELEMETRY INTEGRATION =====

    def enable_telemetry(self, service_name: str = None, endpoint: str = None,
                         console_export: bool = True) -> 'FlowAgentBuilder':
        """Enable OpenTelemetry tracing"""
        if not OTEL_AVAILABLE:
            wprint("OpenTelemetry not available, cannot enable telemetry")
            return self

        self.config.telemetry.enabled = True
        self.config.telemetry.service_name = service_name or self.config.name
        self.config.telemetry.endpoint = endpoint
        self.config.telemetry.console_export = console_export

        # Initialize tracer provider
        self._tracer_provider = TracerProvider()
        trace.set_tracer_provider(self._tracer_provider)

        # Add exporters
        if console_export:
            console_exporter = ConsoleSpanExporter()
            span_processor = BatchSpanProcessor(console_exporter)
            self._tracer_provider.add_span_processor(span_processor)

        if endpoint:
            try:
                otlp_exporter = OTLPSpanExporter(endpoint=endpoint)
                otlp_processor = BatchSpanProcessor(otlp_exporter)
                self._tracer_provider.add_span_processor(otlp_processor)
            except Exception as e:
                wprint(f"Failed to setup OTLP exporter: {e}")

        iprint(f"Telemetry enabled for service: {service_name}")
        return self

    # ===== CHECKPOINT CONFIGURATION =====

    def with_checkpointing(self, enabled: bool = True, interval_seconds: int = 300,
                           checkpoint_dir: str = "./checkpoints", max_checkpoints: int = 10) -> 'FlowAgentBuilder':
        """Configure checkpointing"""
        self.config.checkpoint.enabled = enabled
        self.config.checkpoint.interval_seconds = interval_seconds
        self.config.checkpoint.checkpoint_dir = checkpoint_dir
        self.config.checkpoint.max_checkpoints = max_checkpoints

        if enabled:
            # Ensure checkpoint directory exists
            Path(checkpoint_dir).mkdir(parents=True, exist_ok=True)
            iprint(f"Checkpointing enabled: {checkpoint_dir} (every {interval_seconds}s)")

        return self

    # ===== TOOL MANAGEMENT =====

    def add_tool(self, func: Callable, name: str = None, description: str = None) -> 'FlowAgentBuilder':
        """Add custom tool function"""
        tool_name = name or func.__name__
        self._custom_tools[tool_name] = (func, description or func.__doc__)

        iprint(f"Tool added: {tool_name}")
        return self

    def add_tools_from_module(self, module, prefix: str = "", exclude: list[str] = None) -> 'FlowAgentBuilder':
        """Add all functions from a module as tools"""
        exclude = exclude or []

        for name, obj in inspect.getmembers(module, inspect.isfunction):
            if name in exclude or name.startswith('_'):
                continue

            tool_name = f"{prefix}{name}" if prefix else name
            self.add_tool(obj, name=tool_name)

        iprint(f"Added tools from module {module.__name__}")
        return self

    # ===== PERSONA MANAGEMENT =====

    def add_persona_profile(self, profile_name: str, name: str, style: str = "professional",
                            tone: str = "friendly", personality_traits: list[str] = None,
                            custom_instructions: str = "", response_format: str = None,
                            text_length: str = None) -> 'FlowAgentBuilder':
        """Add a persona profile with optional format configuration"""

        if personality_traits is None:
            personality_traits = ["helpful", "concise"]

        # Create persona config
        persona_data = {
            "name": name,
            "style": style,
            "tone": tone,
            "personality_traits": personality_traits,
            "custom_instructions": custom_instructions,
            "apply_method": "system_prompt",
            "integration_level": "light"
        }

        # Add format config if specified
        if response_format or text_length:
            format_config = {
                "response_format": response_format or "frei-text",
                "text_length": text_length or "chat-conversation",
                "custom_instructions": "",
                "strict_format_adherence": True,
                "quality_threshold": 0.7
            }
            persona_data["format_config"] = format_config

        self.config.persona_profiles[profile_name] = persona_data
        iprint(f"Persona profile added: {profile_name}")
        return self

    def set_active_persona(self, profile_name: str) -> 'FlowAgentBuilder':
        """Set active persona profile"""
        if profile_name in self.config.persona_profiles:
            self.config.active_persona = profile_name
            iprint(f"Active persona set: {profile_name}")
        else:
            wprint(f"Persona profile not found: {profile_name}")
        return self

    def with_developer_persona(self, name: str = "Senior Developer") -> 'FlowAgentBuilder':
        """Add and set a pre-built developer persona"""
        return (self
                .add_persona_profile(
            "developer",
            name=name,
            style="technical",
            tone="professional",
            personality_traits=["precise", "thorough", "security_conscious", "best_practices"],
            custom_instructions="Focus on code quality, maintainability, and security. Always consider edge cases.",
            response_format="code-structure",
            text_length="detailed-indepth"
        )
                .set_active_persona("developer"))

    def with_analyst_persona(self, name: str = "Data Analyst") -> 'FlowAgentBuilder':
        """Add and set a pre-built analyst persona"""
        return (self
                .add_persona_profile(
            "analyst",
            name=name,
            style="analytical",
            tone="objective",
            personality_traits=["methodical", "insight_driven", "evidence_based"],
            custom_instructions="Focus on statistical rigor and actionable recommendations.",
            response_format="with-tables",
            text_length="detailed-indepth"
        )
                .set_active_persona("analyst"))

    def with_assistant_persona(self, name: str = "AI Assistant") -> 'FlowAgentBuilder':
        """Add and set a pre-built general assistant persona"""
        return (self
                .add_persona_profile(
            "assistant",
            name=name,
            style="friendly",
            tone="helpful",
            personality_traits=["helpful", "patient", "clear", "adaptive"],
            custom_instructions="Be helpful and adapt communication to user expertise level.",
            response_format="with-bullet-points",
            text_length="chat-conversation"
        )
                .set_active_persona("assistant"))

    def with_creative_persona(self, name: str = "Creative Assistant") -> 'FlowAgentBuilder':
        """Add and set a pre-built creative persona"""
        return (self
                .add_persona_profile(
            "creative",
            name=name,
            style="creative",
            tone="inspiring",
            personality_traits=["imaginative", "expressive", "innovative", "engaging"],
            custom_instructions="Think outside the box and provide creative, inspiring solutions.",
            response_format="md-text",
            text_length="detailed-indepth"
        )
                .set_active_persona("creative"))

    def with_executive_persona(self, name: str = "Executive Assistant") -> 'FlowAgentBuilder':
        """Add and set a pre-built executive persona"""
        return (self
                .add_persona_profile(
            "executive",
            name=name,
            style="professional",
            tone="authoritative",
            personality_traits=["strategic", "decisive", "results_oriented", "efficient"],
            custom_instructions="Provide strategic insights with executive-level clarity and focus on outcomes.",
            response_format="with-bullet-points",
            text_length="table-conversation"
        )
                .set_active_persona("executive"))

    # ===== VARIABLE MANAGEMENT =====

    def with_custom_variables(self, variables: dict[str, Any]) -> 'FlowAgentBuilder':
        """Add custom variables"""
        self.config.custom_variables.update(variables)
        return self

    def with_world_model(self, world_model: dict[str, Any]) -> 'FlowAgentBuilder':
        """Set initial world model"""
        self.config.initial_world_model.update(world_model)
        return self

    # ===== VALIDATION =====

    def validate_config(self) -> dict[str, list[str]]:
        """Validate the current configuration"""
        issues = {"errors": [], "warnings": []}

        # Validate required settings
        if not self.config.fast_llm_model:
            issues["errors"].append("Fast LLM model not specified")
        if not self.config.complex_llm_model:
            issues["errors"].append("Complex LLM model not specified")

        # Validate MCP configuration
        if self.config.mcp.enabled and not MCP_AVAILABLE:
            issues["errors"].append("MCP enabled but MCP not available")

        # Validate A2A configuration
        if self.config.a2a.enabled and not A2A_AVAILABLE:
            issues["errors"].append("A2A enabled but A2A not available")

        # Validate telemetry
        if self.config.telemetry.enabled and not OTEL_AVAILABLE:
            issues["errors"].append("Telemetry enabled but OpenTelemetry not available")

        # Validate personas
        if self.config.active_persona and self.config.active_persona not in self.config.persona_profiles:
            issues["errors"].append(f"Active persona '{self.config.active_persona}' not found in profiles")

        # Validate checkpoint directory
        if self.config.checkpoint.enabled:
            try:
                Path(self.config.checkpoint.checkpoint_dir).mkdir(parents=True, exist_ok=True)
            except Exception as e:
                issues["warnings"].append(f"Cannot create checkpoint directory: {e}")

        return issues

    # ===== MAIN BUILD METHOD =====

    async def build(self) -> FlowAgent:
        """Build the production-ready FlowAgent"""

        with Spinner(message=f"Building Agent {self.config.name}", symbols='c'):
            iprint(f"Building production FlowAgent: {self.config.name}")

            # Validate configuration
            validation_issues = self.validate_config()
            if validation_issues["errors"]:
                error_msg = f"Configuration validation failed: {', '.join(validation_issues['errors'])}"
                eprint(error_msg)
                raise ValueError(error_msg)

            # Log warnings
            for warning in validation_issues["warnings"]:
                wprint(f"Configuration warning: {warning}")

            try:
                # 1. Setup API configuration
                api_key = None
                if self.config.api_key_env_var:
                    api_key = os.getenv(self.config.api_key_env_var)
                    if not api_key:
                        wprint(f"API key env var {self.config.api_key_env_var} not set")

                # 2. Create persona if configured
                active_persona = None
                if self.config.active_persona and self.config.active_persona in self.config.persona_profiles:
                    persona_data = self.config.persona_profiles[self.config.active_persona]

                    # Create FormatConfig if present
                    format_config = None
                    if "format_config" in persona_data:
                        fc_data = persona_data.pop("format_config")
                        format_config = FormatConfig(
                            response_format=ResponseFormat(fc_data.get("response_format", "frei-text")),
                            text_length=TextLength(fc_data.get("text_length", "chat-conversation")),
                            custom_instructions=fc_data.get("custom_instructions", ""),
                            strict_format_adherence=fc_data.get("strict_format_adherence", True),
                            quality_threshold=fc_data.get("quality_threshold", 0.7)
                        )

                    active_persona = PersonaConfig(**persona_data)
                    active_persona.format_config = format_config

                    iprint(f"Using persona: {active_persona.name}")

                # 3. Create AgentModelData
                amd = AgentModelData(
                    name=self.config.name,
                    fast_llm_model=self.config.fast_llm_model,
                    complex_llm_model=self.config.complex_llm_model,
                    system_message=self.config.system_message,
                    temperature=self.config.temperature,
                    max_tokens=self.config.max_tokens_output,
                    max_input_tokens=self.config.max_tokens_input,
                    api_key=api_key,
                    budget_manager=self._budget_manager,
                    persona=active_persona,
                    use_fast_response=self.config.use_fast_response
                )

                # 4. Create FlowAgent
                agent = FlowAgent(
                    amd=amd,
                    world_model=self.config.initial_world_model.copy(),
                    verbose=self.config.verbose_logging,
                    enable_pause_resume=self.config.checkpoint.enabled,
                    checkpoint_interval=self.config.checkpoint.interval_seconds,
                    max_parallel_tasks=self.config.max_parallel_tasks
                )

                # 5. Add custom variables
                for key, value in self.config.custom_variables.items():
                    agent.set_variable(key, value)

                # 6. Add custom tools
                tools_added = 0
                for tool_name, (tool_func, tool_description) in self._custom_tools.items():
                    try:
                        await agent.add_tool(tool_func, tool_name, tool_description)
                        tools_added += 1
                    except Exception as e:
                        eprint(f"Failed to add tool {tool_name}: {e}")

                with Spinner(message="Loading MCP", symbols='w'):
                    # 6a. Process MCP configuration if needed
                    if hasattr(self, '_mcp_needs_loading') and self._mcp_needs_loading:
                        await self._process_mcp_config()

                # 7. Add MCP tools
                for tool_name, tool_info in self._mcp_tools.items():
                    try:
                        await agent.add_tool(
                            tool_info['function'],
                            tool_name,
                            tool_info['description']
                        )
                        tools_added += 1
                    except Exception as e:
                        eprint(f"Failed to add MCP tool {tool_name}: {e}")

                agent._mcp_session_manager = self._mcp_session_manager

                # 8. Setup MCP server
                if self.config.mcp.enabled and MCP_AVAILABLE:
                    try:
                        agent.setup_mcp_server(
                            host=self.config.mcp.host,
                            port=self.config.mcp.port,
                            name=self.config.mcp.server_name
                        )
                        iprint("MCP server configured")
                    except Exception as e:
                        eprint(f"Failed to setup MCP server: {e}")

                # 9. Setup A2A server
                if self.config.a2a.enabled and A2A_AVAILABLE:
                    try:
                        agent.setup_a2a_server(
                            host=self.config.a2a.host,
                            port=self.config.a2a.port
                        )
                        iprint("A2A server configured")
                    except Exception as e:
                        eprint(f"Failed to setup A2A server: {e}")

                # 10. Initialize enhanced session context
                try:
                    await agent.initialize_session_context(max_history=200)
                    iprint("Enhanced session context initialized")
                except Exception as e:
                    wprint(f"Session context initialization failed: {e}")


                # Final summary
                iprint("ok FlowAgent built successfully!")
                iprint(f"   Agent: {agent.amd.name}")
                iprint(f"   Tools: {tools_added}")
                iprint(f"   MCP: {'ok' if self.config.mcp.enabled else 'F'}")
                iprint(f"   A2A: {'ok' if self.config.a2a.enabled else 'F'}")
                iprint(f"   Telemetry: {'ok' if self.config.telemetry.enabled else 'F'}")
                iprint(f"   Checkpoints: {'ok' if self.config.checkpoint.enabled else 'F'}")
                iprint(f"   Persona: {active_persona.name if active_persona else 'Default'}")

                return agent

            except Exception as e:
                eprint(f"Failed to build FlowAgent: {e}")
                raise

    # ===== FACTORY METHODS =====

    @classmethod
    def create_developer_agent(cls, name: str = "DeveloperAgent",
                               with_mcp: bool = True, with_a2a: bool = False) -> 'FlowAgentBuilder':
        """Create a pre-configured developer agent"""
        builder = (cls()
                   .with_name(name)
                   .with_developer_persona()
                   .with_checkpointing(enabled=True, interval_seconds=300)
                   .verbose(True))

        if with_mcp:
            builder.enable_mcp_server(port=8001)
        if with_a2a:
            builder.enable_a2a_server(port=5001)

        return builder

    @classmethod
    def create_analyst_agent(cls, name: str = "AnalystAgent",
                             with_telemetry: bool = True) -> 'FlowAgentBuilder':
        """Create a pre-configured data analyst agent"""
        builder = (cls()
                   .with_name(name)
                   .with_analyst_persona()
                   .with_checkpointing(enabled=True)
                   .verbose(False))

        if with_telemetry:
            builder.enable_telemetry(console_export=True)

        return builder

    @classmethod
    def create_general_assistant(cls, name: str = "AssistantAgent",
                                 full_integration: bool = True) -> 'FlowAgentBuilder':
        """Create a general-purpose assistant with full integration"""
        builder = (cls()
                   .with_name(name)
                   .with_assistant_persona()
                   .with_checkpointing(enabled=True))

        if full_integration:
            builder.enable_mcp_server()
            builder.enable_a2a_server()
            builder.enable_telemetry()

        return builder

    @classmethod
    def create_creative_agent(cls, name: str = "CreativeAgent") -> 'FlowAgentBuilder':
        """Create a creative assistant agent"""
        return (cls()
                .with_name(name)
                .with_creative_persona()
                .with_temperature(0.8)  # More creative
                .with_checkpointing(enabled=True))

    @classmethod
    def create_executive_agent(cls, name: str = "ExecutiveAgent",
                               with_integrations: bool = True) -> 'FlowAgentBuilder':
        """Create an executive assistant agent"""
        builder = (cls()
                   .with_name(name)
                   .with_executive_persona()
                   .with_checkpointing(enabled=True))

        if with_integrations:
            builder.enable_a2a_server()  # Executives need A2A for delegation
            builder.enable_telemetry()  # Need metrics

        return builder
__init__(config=None, config_path=None)

Initialize builder with configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
def __init__(self, config: AgentConfig = None, config_path: str = None):
    """Initialize builder with configuration"""

    if config and config_path:
        raise ValueError("Provide either config object or config_path, not both")

    if config_path:
        self.config = self.load_config(config_path)
    elif config:
        self.config = config
    else:
        self.config = AgentConfig()

    # Runtime components
    self._custom_tools: dict[str, tuple[Callable, str]] = {}
    self._mcp_tools: dict[str, dict] = {}
    from toolboxv2.mods.isaa.extras.mcp_session_manager import MCPSessionManager

    self._mcp_session_manager = MCPSessionManager()

    self._budget_manager: BudgetManager = None
    self._tracer_provider: TracerProvider = None
    self._a2a_server: Any = None

    # Set logging level
    if self.config.verbose_logging:
        logging.getLogger().setLevel(logging.DEBUG)

    iprint(f"FlowAgent Builder initialized: {self.config.name}")
add_mcp_tool_from_code(name, code, description='')

Add MCP tool from code string

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
973
974
975
976
977
978
979
980
981
def add_mcp_tool_from_code(self, name: str, code: str, description: str = "") -> 'FlowAgentBuilder':
    """Add MCP tool from code string"""
    tool_config = {
        'name': name,
        'description': description,
        'function_code': code
    }
    self._load_direct_mcp_tool(tool_config)
    return self
add_persona_profile(profile_name, name, style='professional', tone='friendly', personality_traits=None, custom_instructions='', response_format=None, text_length=None)

Add a persona profile with optional format configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
def add_persona_profile(self, profile_name: str, name: str, style: str = "professional",
                        tone: str = "friendly", personality_traits: list[str] = None,
                        custom_instructions: str = "", response_format: str = None,
                        text_length: str = None) -> 'FlowAgentBuilder':
    """Add a persona profile with optional format configuration"""

    if personality_traits is None:
        personality_traits = ["helpful", "concise"]

    # Create persona config
    persona_data = {
        "name": name,
        "style": style,
        "tone": tone,
        "personality_traits": personality_traits,
        "custom_instructions": custom_instructions,
        "apply_method": "system_prompt",
        "integration_level": "light"
    }

    # Add format config if specified
    if response_format or text_length:
        format_config = {
            "response_format": response_format or "frei-text",
            "text_length": text_length or "chat-conversation",
            "custom_instructions": "",
            "strict_format_adherence": True,
            "quality_threshold": 0.7
        }
        persona_data["format_config"] = format_config

    self.config.persona_profiles[profile_name] = persona_data
    iprint(f"Persona profile added: {profile_name}")
    return self
add_tool(func, name=None, description=None)

Add custom tool function

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1055
1056
1057
1058
1059
1060
1061
def add_tool(self, func: Callable, name: str = None, description: str = None) -> 'FlowAgentBuilder':
    """Add custom tool function"""
    tool_name = name or func.__name__
    self._custom_tools[tool_name] = (func, description or func.__doc__)

    iprint(f"Tool added: {tool_name}")
    return self
add_tools_from_module(module, prefix='', exclude=None)

Add all functions from a module as tools

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
def add_tools_from_module(self, module, prefix: str = "", exclude: list[str] = None) -> 'FlowAgentBuilder':
    """Add all functions from a module as tools"""
    exclude = exclude or []

    for name, obj in inspect.getmembers(module, inspect.isfunction):
        if name in exclude or name.startswith('_'):
            continue

        tool_name = f"{prefix}{name}" if prefix else name
        self.add_tool(obj, name=tool_name)

    iprint(f"Added tools from module {module.__name__}")
    return self
build() async

Build the production-ready FlowAgent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
async def build(self) -> FlowAgent:
    """Build the production-ready FlowAgent"""

    with Spinner(message=f"Building Agent {self.config.name}", symbols='c'):
        iprint(f"Building production FlowAgent: {self.config.name}")

        # Validate configuration
        validation_issues = self.validate_config()
        if validation_issues["errors"]:
            error_msg = f"Configuration validation failed: {', '.join(validation_issues['errors'])}"
            eprint(error_msg)
            raise ValueError(error_msg)

        # Log warnings
        for warning in validation_issues["warnings"]:
            wprint(f"Configuration warning: {warning}")

        try:
            # 1. Setup API configuration
            api_key = None
            if self.config.api_key_env_var:
                api_key = os.getenv(self.config.api_key_env_var)
                if not api_key:
                    wprint(f"API key env var {self.config.api_key_env_var} not set")

            # 2. Create persona if configured
            active_persona = None
            if self.config.active_persona and self.config.active_persona in self.config.persona_profiles:
                persona_data = self.config.persona_profiles[self.config.active_persona]

                # Create FormatConfig if present
                format_config = None
                if "format_config" in persona_data:
                    fc_data = persona_data.pop("format_config")
                    format_config = FormatConfig(
                        response_format=ResponseFormat(fc_data.get("response_format", "frei-text")),
                        text_length=TextLength(fc_data.get("text_length", "chat-conversation")),
                        custom_instructions=fc_data.get("custom_instructions", ""),
                        strict_format_adherence=fc_data.get("strict_format_adherence", True),
                        quality_threshold=fc_data.get("quality_threshold", 0.7)
                    )

                active_persona = PersonaConfig(**persona_data)
                active_persona.format_config = format_config

                iprint(f"Using persona: {active_persona.name}")

            # 3. Create AgentModelData
            amd = AgentModelData(
                name=self.config.name,
                fast_llm_model=self.config.fast_llm_model,
                complex_llm_model=self.config.complex_llm_model,
                system_message=self.config.system_message,
                temperature=self.config.temperature,
                max_tokens=self.config.max_tokens_output,
                max_input_tokens=self.config.max_tokens_input,
                api_key=api_key,
                budget_manager=self._budget_manager,
                persona=active_persona,
                use_fast_response=self.config.use_fast_response
            )

            # 4. Create FlowAgent
            agent = FlowAgent(
                amd=amd,
                world_model=self.config.initial_world_model.copy(),
                verbose=self.config.verbose_logging,
                enable_pause_resume=self.config.checkpoint.enabled,
                checkpoint_interval=self.config.checkpoint.interval_seconds,
                max_parallel_tasks=self.config.max_parallel_tasks
            )

            # 5. Add custom variables
            for key, value in self.config.custom_variables.items():
                agent.set_variable(key, value)

            # 6. Add custom tools
            tools_added = 0
            for tool_name, (tool_func, tool_description) in self._custom_tools.items():
                try:
                    await agent.add_tool(tool_func, tool_name, tool_description)
                    tools_added += 1
                except Exception as e:
                    eprint(f"Failed to add tool {tool_name}: {e}")

            with Spinner(message="Loading MCP", symbols='w'):
                # 6a. Process MCP configuration if needed
                if hasattr(self, '_mcp_needs_loading') and self._mcp_needs_loading:
                    await self._process_mcp_config()

            # 7. Add MCP tools
            for tool_name, tool_info in self._mcp_tools.items():
                try:
                    await agent.add_tool(
                        tool_info['function'],
                        tool_name,
                        tool_info['description']
                    )
                    tools_added += 1
                except Exception as e:
                    eprint(f"Failed to add MCP tool {tool_name}: {e}")

            agent._mcp_session_manager = self._mcp_session_manager

            # 8. Setup MCP server
            if self.config.mcp.enabled and MCP_AVAILABLE:
                try:
                    agent.setup_mcp_server(
                        host=self.config.mcp.host,
                        port=self.config.mcp.port,
                        name=self.config.mcp.server_name
                    )
                    iprint("MCP server configured")
                except Exception as e:
                    eprint(f"Failed to setup MCP server: {e}")

            # 9. Setup A2A server
            if self.config.a2a.enabled and A2A_AVAILABLE:
                try:
                    agent.setup_a2a_server(
                        host=self.config.a2a.host,
                        port=self.config.a2a.port
                    )
                    iprint("A2A server configured")
                except Exception as e:
                    eprint(f"Failed to setup A2A server: {e}")

            # 10. Initialize enhanced session context
            try:
                await agent.initialize_session_context(max_history=200)
                iprint("Enhanced session context initialized")
            except Exception as e:
                wprint(f"Session context initialization failed: {e}")


            # Final summary
            iprint("ok FlowAgent built successfully!")
            iprint(f"   Agent: {agent.amd.name}")
            iprint(f"   Tools: {tools_added}")
            iprint(f"   MCP: {'ok' if self.config.mcp.enabled else 'F'}")
            iprint(f"   A2A: {'ok' if self.config.a2a.enabled else 'F'}")
            iprint(f"   Telemetry: {'ok' if self.config.telemetry.enabled else 'F'}")
            iprint(f"   Checkpoints: {'ok' if self.config.checkpoint.enabled else 'F'}")
            iprint(f"   Persona: {active_persona.name if active_persona else 'Default'}")

            return agent

        except Exception as e:
            eprint(f"Failed to build FlowAgent: {e}")
            raise
create_analyst_agent(name='AnalystAgent', with_telemetry=True) classmethod

Create a pre-configured data analyst agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
@classmethod
def create_analyst_agent(cls, name: str = "AnalystAgent",
                         with_telemetry: bool = True) -> 'FlowAgentBuilder':
    """Create a pre-configured data analyst agent"""
    builder = (cls()
               .with_name(name)
               .with_analyst_persona()
               .with_checkpointing(enabled=True)
               .verbose(False))

    if with_telemetry:
        builder.enable_telemetry(console_export=True)

    return builder
create_creative_agent(name='CreativeAgent') classmethod

Create a creative assistant agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1450
1451
1452
1453
1454
1455
1456
1457
@classmethod
def create_creative_agent(cls, name: str = "CreativeAgent") -> 'FlowAgentBuilder':
    """Create a creative assistant agent"""
    return (cls()
            .with_name(name)
            .with_creative_persona()
            .with_temperature(0.8)  # More creative
            .with_checkpointing(enabled=True))
create_developer_agent(name='DeveloperAgent', with_mcp=True, with_a2a=False) classmethod

Create a pre-configured developer agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
@classmethod
def create_developer_agent(cls, name: str = "DeveloperAgent",
                           with_mcp: bool = True, with_a2a: bool = False) -> 'FlowAgentBuilder':
    """Create a pre-configured developer agent"""
    builder = (cls()
               .with_name(name)
               .with_developer_persona()
               .with_checkpointing(enabled=True, interval_seconds=300)
               .verbose(True))

    if with_mcp:
        builder.enable_mcp_server(port=8001)
    if with_a2a:
        builder.enable_a2a_server(port=5001)

    return builder
create_executive_agent(name='ExecutiveAgent', with_integrations=True) classmethod

Create an executive assistant agent

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
@classmethod
def create_executive_agent(cls, name: str = "ExecutiveAgent",
                           with_integrations: bool = True) -> 'FlowAgentBuilder':
    """Create an executive assistant agent"""
    builder = (cls()
               .with_name(name)
               .with_executive_persona()
               .with_checkpointing(enabled=True))

    if with_integrations:
        builder.enable_a2a_server()  # Executives need A2A for delegation
        builder.enable_telemetry()  # Need metrics

    return builder
create_general_assistant(name='AssistantAgent', full_integration=True) classmethod

Create a general-purpose assistant with full integration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
@classmethod
def create_general_assistant(cls, name: str = "AssistantAgent",
                             full_integration: bool = True) -> 'FlowAgentBuilder':
    """Create a general-purpose assistant with full integration"""
    builder = (cls()
               .with_name(name)
               .with_assistant_persona()
               .with_checkpointing(enabled=True))

    if full_integration:
        builder.enable_mcp_server()
        builder.enable_a2a_server()
        builder.enable_telemetry()

    return builder
enable_a2a_server(host='0.0.0.0', port=5000, agent_name=None, agent_description=None)

Enable A2A server for agent-to-agent communication

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
def enable_a2a_server(self, host: str = "0.0.0.0", port: int = 5000,
                      agent_name: str = None, agent_description: str = None) -> 'FlowAgentBuilder':
    """Enable A2A server for agent-to-agent communication"""
    if not A2A_AVAILABLE:
        wprint("A2A not available, cannot enable server")
        return self

    self.config.a2a.enabled = True
    self.config.a2a.host = host
    self.config.a2a.port = port
    self.config.a2a.agent_name = agent_name or self.config.name
    self.config.a2a.agent_description = agent_description or self.config.description

    iprint(f"A2A server enabled: {host}:{port}")
    return self
enable_mcp_server(host='0.0.0.0', port=8000, server_name=None)

Enable MCP server

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
303
304
305
306
307
308
309
310
311
312
313
314
315
316
def enable_mcp_server(self, host: str = "0.0.0.0", port: int = 8000,
                      server_name: str = None) -> 'FlowAgentBuilder':
    """Enable MCP server"""
    if not MCP_AVAILABLE:
        wprint("MCP not available, cannot enable server")
        return self

    self.config.mcp.enabled = True
    self.config.mcp.host = host
    self.config.mcp.port = port
    self.config.mcp.server_name = server_name or f"{self.config.name}_MCP"

    iprint(f"MCP server enabled: {host}:{port}")
    return self
enable_telemetry(service_name=None, endpoint=None, console_export=True)

Enable OpenTelemetry tracing

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
def enable_telemetry(self, service_name: str = None, endpoint: str = None,
                     console_export: bool = True) -> 'FlowAgentBuilder':
    """Enable OpenTelemetry tracing"""
    if not OTEL_AVAILABLE:
        wprint("OpenTelemetry not available, cannot enable telemetry")
        return self

    self.config.telemetry.enabled = True
    self.config.telemetry.service_name = service_name or self.config.name
    self.config.telemetry.endpoint = endpoint
    self.config.telemetry.console_export = console_export

    # Initialize tracer provider
    self._tracer_provider = TracerProvider()
    trace.set_tracer_provider(self._tracer_provider)

    # Add exporters
    if console_export:
        console_exporter = ConsoleSpanExporter()
        span_processor = BatchSpanProcessor(console_exporter)
        self._tracer_provider.add_span_processor(span_processor)

    if endpoint:
        try:
            otlp_exporter = OTLPSpanExporter(endpoint=endpoint)
            otlp_processor = BatchSpanProcessor(otlp_exporter)
            self._tracer_provider.add_span_processor(otlp_processor)
        except Exception as e:
            wprint(f"Failed to setup OTLP exporter: {e}")

    iprint(f"Telemetry enabled for service: {service_name}")
    return self
from_config_file(config_path) classmethod

Create builder from configuration file

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
256
257
258
259
@classmethod
def from_config_file(cls, config_path: str) -> 'FlowAgentBuilder':
    """Create builder from configuration file"""
    return cls(config_path=config_path)
load_config(config_path)

Load agent configuration from file

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
def load_config(self, config_path: str) -> AgentConfig:
    """Load agent configuration from file"""
    path = Path(config_path)
    if not path.exists():
        raise FileNotFoundError(f"Config file not found: {config_path}")

    try:
        with open(path, encoding='utf-8') as f:
            if path.suffix.lower() in ['.yaml', '.yml']:
                data = yaml.safe_load(f)
            else:
                data = json.load(f)

        return AgentConfig(**data)

    except Exception as e:
        eprint(f"Failed to load config from {config_path}: {e}")
        raise
load_mcp_tools_from_config(config_path)

Enhanced MCP config loading with automatic session management and full capability extraction

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
def load_mcp_tools_from_config(self, config_path: str | dict) -> 'FlowAgentBuilder':
    """Enhanced MCP config loading with automatic session management and full capability extraction"""
    if not MCP_AVAILABLE:
        wprint("MCP not available, skipping tool loading")
        return self

    if isinstance(config_path, dict):
        mcp_config = config_path
        from toolboxv2 import get_app
        name = self.config.name or "inline_config"
        path = Path(get_app().appdata) / "isaa" / "MCPConfig" / f"{name}.json"
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(json.dumps(mcp_config, indent=2))
        config_path = path
    else:
        config_path = Path(config_path)
        if not config_path.exists():
            raise FileNotFoundError(f"MCP config not found: {config_path}")

        try:
            with open(config_path, encoding='utf-8') as f:
                if config_path.suffix.lower() in ['.yaml', '.yml']:
                    mcp_config = yaml.safe_load(f)
                else:
                    mcp_config = json.load(f)

        except Exception as e:
            eprint(f"Failed to load MCP config from {config_path}: {e}")
            raise

    # Store config for async processing
    self._mcp_config_data = mcp_config
    self.config.mcp.config_path = str(config_path)

    # Mark for processing during build
    self._mcp_needs_loading = True

    iprint(f"MCP config loaded from {config_path}, will process during build")

    return self
save_config(config_path, format='yaml')

Save current configuration to file

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
def save_config(self, config_path: str, format: str = 'yaml'):
    """Save current configuration to file"""
    path = Path(config_path)
    path.parent.mkdir(parents=True, exist_ok=True)

    try:
        data = self.config.model_dump()

        with open(path, 'w', encoding='utf-8') as f:
            if format.lower() == 'yaml':
                yaml.dump(data, f, default_flow_style=False, indent=2)
            else:
                json.dump(data, f, indent=2)

        iprint(f"Configuration saved to {config_path}")

    except Exception as e:
        eprint(f"Failed to save config to {config_path}: {e}")
        raise
set_active_persona(profile_name)

Set active persona profile

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1114
1115
1116
1117
1118
1119
1120
1121
def set_active_persona(self, profile_name: str) -> 'FlowAgentBuilder':
    """Set active persona profile"""
    if profile_name in self.config.persona_profiles:
        self.config.active_persona = profile_name
        iprint(f"Active persona set: {profile_name}")
    else:
        wprint(f"Persona profile not found: {profile_name}")
    return self
validate_config()

Validate the current configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
def validate_config(self) -> dict[str, list[str]]:
    """Validate the current configuration"""
    issues = {"errors": [], "warnings": []}

    # Validate required settings
    if not self.config.fast_llm_model:
        issues["errors"].append("Fast LLM model not specified")
    if not self.config.complex_llm_model:
        issues["errors"].append("Complex LLM model not specified")

    # Validate MCP configuration
    if self.config.mcp.enabled and not MCP_AVAILABLE:
        issues["errors"].append("MCP enabled but MCP not available")

    # Validate A2A configuration
    if self.config.a2a.enabled and not A2A_AVAILABLE:
        issues["errors"].append("A2A enabled but A2A not available")

    # Validate telemetry
    if self.config.telemetry.enabled and not OTEL_AVAILABLE:
        issues["errors"].append("Telemetry enabled but OpenTelemetry not available")

    # Validate personas
    if self.config.active_persona and self.config.active_persona not in self.config.persona_profiles:
        issues["errors"].append(f"Active persona '{self.config.active_persona}' not found in profiles")

    # Validate checkpoint directory
    if self.config.checkpoint.enabled:
        try:
            Path(self.config.checkpoint.checkpoint_dir).mkdir(parents=True, exist_ok=True)
        except Exception as e:
            issues["warnings"].append(f"Cannot create checkpoint directory: {e}")

    return issues
verbose(enable=True)

Enable verbose logging

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
294
295
296
297
298
299
def verbose(self, enable: bool = True) -> 'FlowAgentBuilder':
    """Enable verbose logging"""
    self.config.verbose_logging = enable
    if enable:
        logging.getLogger().setLevel(logging.DEBUG)
    return self
with_analyst_persona(name='Data Analyst')

Add and set a pre-built analyst persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
def with_analyst_persona(self, name: str = "Data Analyst") -> 'FlowAgentBuilder':
    """Add and set a pre-built analyst persona"""
    return (self
            .add_persona_profile(
        "analyst",
        name=name,
        style="analytical",
        tone="objective",
        personality_traits=["methodical", "insight_driven", "evidence_based"],
        custom_instructions="Focus on statistical rigor and actionable recommendations.",
        response_format="with-tables",
        text_length="detailed-indepth"
    )
            .set_active_persona("analyst"))
with_assistant_persona(name='AI Assistant')

Add and set a pre-built general assistant persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
def with_assistant_persona(self, name: str = "AI Assistant") -> 'FlowAgentBuilder':
    """Add and set a pre-built general assistant persona"""
    return (self
            .add_persona_profile(
        "assistant",
        name=name,
        style="friendly",
        tone="helpful",
        personality_traits=["helpful", "patient", "clear", "adaptive"],
        custom_instructions="Be helpful and adapt communication to user expertise level.",
        response_format="with-bullet-points",
        text_length="chat-conversation"
    )
            .set_active_persona("assistant"))
with_budget_manager(max_cost=10.0)

Enable budget management

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
285
286
287
288
289
290
291
292
def with_budget_manager(self, max_cost: float = 10.0) -> 'FlowAgentBuilder':
    """Enable budget management"""
    if LITELLM_AVAILABLE:
        self._budget_manager = BudgetManager("agent")
        iprint(f"Budget manager enabled: ${max_cost}")
    else:
        wprint("LiteLLM not available, budget manager disabled")
    return self
with_checkpointing(enabled=True, interval_seconds=300, checkpoint_dir='./checkpoints', max_checkpoints=10)

Configure checkpointing

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
def with_checkpointing(self, enabled: bool = True, interval_seconds: int = 300,
                       checkpoint_dir: str = "./checkpoints", max_checkpoints: int = 10) -> 'FlowAgentBuilder':
    """Configure checkpointing"""
    self.config.checkpoint.enabled = enabled
    self.config.checkpoint.interval_seconds = interval_seconds
    self.config.checkpoint.checkpoint_dir = checkpoint_dir
    self.config.checkpoint.max_checkpoints = max_checkpoints

    if enabled:
        # Ensure checkpoint directory exists
        Path(checkpoint_dir).mkdir(parents=True, exist_ok=True)
        iprint(f"Checkpointing enabled: {checkpoint_dir} (every {interval_seconds}s)")

    return self
with_creative_persona(name='Creative Assistant')

Add and set a pre-built creative persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
def with_creative_persona(self, name: str = "Creative Assistant") -> 'FlowAgentBuilder':
    """Add and set a pre-built creative persona"""
    return (self
            .add_persona_profile(
        "creative",
        name=name,
        style="creative",
        tone="inspiring",
        personality_traits=["imaginative", "expressive", "innovative", "engaging"],
        custom_instructions="Think outside the box and provide creative, inspiring solutions.",
        response_format="md-text",
        text_length="detailed-indepth"
    )
            .set_active_persona("creative"))
with_custom_variables(variables)

Add custom variables

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1200
1201
1202
1203
def with_custom_variables(self, variables: dict[str, Any]) -> 'FlowAgentBuilder':
    """Add custom variables"""
    self.config.custom_variables.update(variables)
    return self
with_developer_persona(name='Senior Developer')

Add and set a pre-built developer persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
def with_developer_persona(self, name: str = "Senior Developer") -> 'FlowAgentBuilder':
    """Add and set a pre-built developer persona"""
    return (self
            .add_persona_profile(
        "developer",
        name=name,
        style="technical",
        tone="professional",
        personality_traits=["precise", "thorough", "security_conscious", "best_practices"],
        custom_instructions="Focus on code quality, maintainability, and security. Always consider edge cases.",
        response_format="code-structure",
        text_length="detailed-indepth"
    )
            .set_active_persona("developer"))
with_executive_persona(name='Executive Assistant')

Add and set a pre-built executive persona

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
def with_executive_persona(self, name: str = "Executive Assistant") -> 'FlowAgentBuilder':
    """Add and set a pre-built executive persona"""
    return (self
            .add_persona_profile(
        "executive",
        name=name,
        style="professional",
        tone="authoritative",
        personality_traits=["strategic", "decisive", "results_oriented", "efficient"],
        custom_instructions="Provide strategic insights with executive-level clarity and focus on outcomes.",
        response_format="with-bullet-points",
        text_length="table-conversation"
    )
            .set_active_persona("executive"))
with_models(fast_model, complex_model=None)

Set LLM models

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
268
269
270
271
272
273
def with_models(self, fast_model: str, complex_model: str = None) -> 'FlowAgentBuilder':
    """Set LLM models"""
    self.config.fast_llm_model = fast_model
    if complex_model:
        self.config.complex_llm_model = complex_model
    return self
with_name(name)

Set agent name

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
263
264
265
266
def with_name(self, name: str) -> 'FlowAgentBuilder':
    """Set agent name"""
    self.config.name = name
    return self
with_system_message(message)

Set system message

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
275
276
277
278
def with_system_message(self, message: str) -> 'FlowAgentBuilder':
    """Set system message"""
    self.config.system_message = message
    return self
with_temperature(temp)

Set temperature

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
280
281
282
283
def with_temperature(self, temp: float) -> 'FlowAgentBuilder':
    """Set temperature"""
    self.config.temperature = temp
    return self
with_world_model(world_model)

Set initial world model

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1205
1206
1207
1208
def with_world_model(self, world_model: dict[str, Any]) -> 'FlowAgentBuilder':
    """Set initial world model"""
    self.config.initial_world_model.update(world_model)
    return self
MCPConfig

Bases: BaseModel

MCP server and tools configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
87
88
89
90
91
92
93
94
95
96
97
class MCPConfig(BaseModel):
    """MCP server and tools configuration"""
    model_config = ConfigDict(arbitrary_types_allowed=True)

    enabled: bool = False
    config_path: Optional[str] = None  # Path to MCP tools config file
    server_name: Optional[str] = None
    host: str = "0.0.0.0"
    port: int = 8000
    auto_expose_tools: bool = True
    tools_from_config: list[dict[str, Any]] = Field(default_factory=list)
TelemetryConfig

Bases: BaseModel

OpenTelemetry configuration

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
113
114
115
116
117
118
119
120
class TelemetryConfig(BaseModel):
    """OpenTelemetry configuration"""
    enabled: bool = False
    service_name: Optional[str] = None
    endpoint: Optional[str] = None  # OTLP endpoint
    console_export: bool = True
    batch_export: bool = True
    sample_rate: float = 1.0
detect_shell()

Detects the best available shell and the argument to execute a command. Returns: A tuple of (shell_executable, command_argument). e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def detect_shell() -> tuple[str, str]:
    """
    Detects the best available shell and the argument to execute a command.
    Returns:
        A tuple of (shell_executable, command_argument).
        e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')
    """
    if platform.system() == "Windows":
        if shell_path := shutil.which("pwsh"):
            return shell_path, "-Command"
        if shell_path := shutil.which("powershell"):
            return shell_path, "-Command"
        return "cmd.exe", "/c"

    shell_env = os.environ.get("SHELL")
    if shell_env and shutil.which(shell_env):
        return shell_env, "-c"

    for shell in ["bash", "zsh", "sh"]:
        if shell_path := shutil.which(shell):
            return shell_path, "-c"

    return "/bin/sh", "-c"
example_production_usage() async

Production usage example with full features

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
async def example_production_usage():
    """Production usage example with full features"""

    iprint("=== Production FlowAgent Builder Example ===")

    # Example 1: Developer agent with full MCP integration
    iprint("Creating developer agent with MCP integration...")

    # Add a custom tool
    def get_system_info():
        """Get basic system information"""
        import platform
        return {
            "platform": platform.platform(),
            "python_version": platform.python_version(),
            "architecture": platform.architecture()
        }

    developer_agent = await (FlowAgentBuilder
                             .create_developer_agent("ProductionDev", with_mcp=True, with_a2a=True)
                             .add_tool(get_system_info, "get_system_info", "Get system information")
                             .enable_telemetry(console_export=True)
                             .with_custom_variables({
        "project_name": "FlowAgent Production",
        "environment": "production"
    })
                             .build())

    # Test the developer agent
    dev_response = await developer_agent.a_run(
        "Hello! I'm working on {{ project_name }}. Can you tell me about the system and create a simple Python function?"
    )
    iprint(f"Developer agent response: {dev_response[:200]}...")

    # Example 2: Load from configuration file
    iprint("\nTesting configuration save/load...")

    # Save current config
    config_path = "/tmp/production_agent_config.yaml"
    builder = FlowAgentBuilder.create_analyst_agent("ConfigTestAgent")
    builder.save_config(config_path)

    # Load from config
    loaded_builder = FlowAgentBuilder.from_config_file(config_path)
    config_agent = await loaded_builder.build()

    config_response = await config_agent.a_run("Analyze this data: [1, 2, 3, 4, 5]")
    iprint(f"Config-loaded agent response: {config_response[:150]}...")

    # Example 3: Agent with MCP tools from config
    iprint("\nTesting MCP tools integration...")

    # Create a sample MCP config
    mcp_config = {
        "tools": [
            {
                "name": "weather_checker",
                "description": "Check weather for a location",
                "function_code": '''
async def weather_checker(location: str) -> str:
    """Mock weather checker"""
    import random
    conditions = ["sunny", "cloudy", "rainy", "snowy"]
    temp = random.randint(-10, 35)
    condition = random.choice(conditions)
    return f"Weather in {location}: {condition}, {temp}°C"
'''
            }
        ]
    }

    mcp_config_path = "/tmp/mcp_tools_config.json"
    with open(mcp_config_path, 'w') as f:
        json.dump(mcp_config, f, indent=2)

    mcp_agent = await (FlowAgentBuilder()
                       .with_name("MCPTestAgent")
                       .with_assistant_persona()
                       .enable_mcp_server(port=8002)
                       .load_mcp_tools_from_config(mcp_config_path)
                       .build())

    mcp_response = await mcp_agent.a_run("What's the weather like in Berlin?")
    iprint(f"MCP agent response: {mcp_response[:150]}...")

    # Show agent status
    iprint("\n=== Agent Status ===")
    status = developer_agent.status(pretty_print=False)
    iprint(f"Developer agent tools: {len(status['capabilities']['tool_names'])}")
    iprint(f"MCP agent tools: {len(mcp_agent.shared.get('available_tools', []))}")

    # Cleanup
    await developer_agent.close()
    await config_agent.close()
    await mcp_agent.close()

    iprint("Production example completed successfully!")
example_quick_start() async

Quick start examples for common scenarios

Source code in toolboxv2/mods/isaa/base/Agent/builder.py
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
async def example_quick_start():
    """Quick start examples for common scenarios"""

    iprint("=== Quick Start Examples ===")

    # 1. Simple developer agent
    dev_agent = await FlowAgentBuilder.create_developer_agent("QuickDev").build()
    response1 = await dev_agent.a_run("Create a Python function to validate email addresses")
    iprint(f"Quick dev response: {response1[:100]}...")
    await dev_agent.close()

    # 2. Analyst with custom data
    analyst_agent = await (FlowAgentBuilder
                           .create_analyst_agent("QuickAnalyst")
                           .with_custom_variables({"dataset": "sales_data_2024"})
                           .build())
    response2 = await analyst_agent.a_run("Analyze the trends in {{ dataset }}")
    iprint(f"Quick analyst response: {response2[:100]}...")
    await analyst_agent.close()

    # 3. Creative assistant
    creative_agent = await FlowAgentBuilder.create_creative_agent("QuickCreative").build()
    response3 = await creative_agent.a_run("Write a creative story about AI agents collaborating")
    iprint(f"Quick creative response: {response3[:100]}...")
    await creative_agent.close()

    iprint("Quick start examples completed!")
chain
CF

Chain Format - handles formatting and data extraction between tasks.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
class CF:
    """Chain Format - handles formatting and data extraction between tasks."""

    def __init__(self, format_class: type[BaseModel]):
        self.format_class = format_class
        self.extract_key: str | tuple | None = None
        self.is_parallel_extraction = False

    def __sub__(self, key: str | tuple):
        """Implements the - operator for data extraction keys."""
        new_cf = copy.copy(self)
        if isinstance(key, str):
            if '[n]' in key:
                new_cf.extract_key = key.replace('[n]', '')
                new_cf.is_parallel_extraction = True
            else:
                new_cf.extract_key = key
        elif isinstance(key, tuple):
            new_cf.extract_key = key
        return new_cf
__sub__(key)

Implements the - operator for data extraction keys.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
24
25
26
27
28
29
30
31
32
33
34
35
def __sub__(self, key: str | tuple):
    """Implements the - operator for data extraction keys."""
    new_cf = copy.copy(self)
    if isinstance(key, str):
        if '[n]' in key:
            new_cf.extract_key = key.replace('[n]', '')
            new_cf.is_parallel_extraction = True
        else:
            new_cf.extract_key = key
    elif isinstance(key, tuple):
        new_cf.extract_key = key
    return new_cf
Chain

Bases: ChainBase

The main class for creating and executing sequential chains of tasks.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
class Chain(ChainBase):
    """The main class for creating and executing sequential chains of tasks."""

    def __init__(self, agent: 'FlowAgent' = None):
        self.tasks: list[Any] = [agent] if agent else []
        self.progress_tracker: ChainPrinter | None = None

    @classmethod
    def _create_chain(cls, components: list[Any]) -> 'Chain':
        chain = cls()
        chain.tasks = components
        return chain

    def _extract_data(self, data: dict, cf: CF) -> Any:
        """Extracts data from a dictionary based on the CF configuration."""
        if not isinstance(data, dict):
            return data

        key = cf.extract_key
        if key == '*':
            return data
        if isinstance(key, tuple):
            return {k: data.get(k) for k in key if k in data}
        if isinstance(key, str) and key in data:
            return data[key]
        return data  # Return original data if key not found

    async def a_run(self, query: Any, **kwargs):
        """
        Executes the chain of tasks asynchronously with dynamic method selection,
        data extraction, and auto-parallelization.
        """
        current_data = query

        # We need to iterate with an index to look ahead
        i = 0
        while i < len(self.tasks):
            task = self.tasks[i]

            # --- Auto-Erkennung und Ausführung ---
            if hasattr(task, 'a_run') and hasattr(task, 'a_format_class'):
                next_task = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                task.active_session = kwargs.get("session_id", "default")
                # Dynamische Entscheidung: a_format_class oder a_run aufrufen?
                if isinstance(next_task, CF):
                    # Nächste Aufgabe ist Formatierung, also a_format_class aufrufen
                    current_data = await task.a_format_class(
                        next_task.format_class, str(current_data), **kwargs
                    )
                else:
                    # Standardausführung
                    current_data = await task.a_run(str(current_data), **kwargs)
                task.active_session = None

            elif isinstance(task, CF):
                # --- Auto-Extraktion und Parallelisierung ---
                if task.extract_key:
                    extracted_data = self._extract_data(current_data, task)

                    if task.is_parallel_extraction and isinstance(extracted_data, list):
                        next_task_for_parallel = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                        if next_task_for_parallel:
                            # Erstelle eine temporäre Parallel-Kette und führe sie aus
                            parallel_runner = ParallelChain([next_task_for_parallel] * len(extracted_data))

                            # Führe jeden Task mit dem entsprechenden Datenelement aus
                            parallel_tasks = [
                                next_task_for_parallel.a_run(item, **kwargs) for item in extracted_data
                            ]
                            current_data = await asyncio.gather(*parallel_tasks)

                            print("Parallel results:", type(current_data))
                            print("Parallel results:", len(current_data))
                            # Überspringe die nächste Aufgabe, da sie bereits parallel ausgeführt wurde
                            i += 1
                        else:
                            current_data = extracted_data
                    else:
                        current_data = extracted_data
                else:
                    # Keine Extraktion, Daten bleiben unverändert (CF dient nur als Marker)
                    pass

            elif isinstance(task, ParallelChain | ConditionalChain | ErrorHandlingChain):
                current_data = await task.a_run(current_data, **kwargs)

            elif callable(task) and not isinstance(task, (ChainBase, type)):
                # Check if the function is async, then await it
                if asyncio.iscoroutinefunction(task):
                    current_data = await task(current_data)
                # Otherwise, run the synchronous function normally
                else:
                    current_data = task(current_data)
            elif hasattr(task, 'a_run'):
                current_data = await task.a_run(current_data, **kwargs)
            elif isinstance(task, IS):
                # IS needs to be paired with >> to form a ConditionalChain
                next_task_for_cond = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                if next_task_for_cond:
                    # Form a conditional chain on the fly
                    conditional_task = ConditionalChain(task, next_task_for_cond)
                    # Check for a false branch defined with %
                    next_next_task = self.tasks[i + 2] if (i + 2) < len(self.tasks) else None
                    if isinstance(next_next_task, ConditionalChain) and next_next_task.false_branch:
                        conditional_task.false_branch = next_next_task.false_branch
                        i += 1  # also skip the false branch marker

                    current_data = await conditional_task.a_run(current_data, **kwargs)
                    i += 1  # Skip the next task as it's part of the conditional
                else:
                    raise ValueError("IS condition must be followed by a task to execute.")

            i += 1  # Gehe zur nächsten Aufgabe

        return current_data
a_run(query, **kwargs) async

Executes the chain of tasks asynchronously with dynamic method selection, data extraction, and auto-parallelization.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
async def a_run(self, query: Any, **kwargs):
    """
    Executes the chain of tasks asynchronously with dynamic method selection,
    data extraction, and auto-parallelization.
    """
    current_data = query

    # We need to iterate with an index to look ahead
    i = 0
    while i < len(self.tasks):
        task = self.tasks[i]

        # --- Auto-Erkennung und Ausführung ---
        if hasattr(task, 'a_run') and hasattr(task, 'a_format_class'):
            next_task = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
            task.active_session = kwargs.get("session_id", "default")
            # Dynamische Entscheidung: a_format_class oder a_run aufrufen?
            if isinstance(next_task, CF):
                # Nächste Aufgabe ist Formatierung, also a_format_class aufrufen
                current_data = await task.a_format_class(
                    next_task.format_class, str(current_data), **kwargs
                )
            else:
                # Standardausführung
                current_data = await task.a_run(str(current_data), **kwargs)
            task.active_session = None

        elif isinstance(task, CF):
            # --- Auto-Extraktion und Parallelisierung ---
            if task.extract_key:
                extracted_data = self._extract_data(current_data, task)

                if task.is_parallel_extraction and isinstance(extracted_data, list):
                    next_task_for_parallel = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
                    if next_task_for_parallel:
                        # Erstelle eine temporäre Parallel-Kette und führe sie aus
                        parallel_runner = ParallelChain([next_task_for_parallel] * len(extracted_data))

                        # Führe jeden Task mit dem entsprechenden Datenelement aus
                        parallel_tasks = [
                            next_task_for_parallel.a_run(item, **kwargs) for item in extracted_data
                        ]
                        current_data = await asyncio.gather(*parallel_tasks)

                        print("Parallel results:", type(current_data))
                        print("Parallel results:", len(current_data))
                        # Überspringe die nächste Aufgabe, da sie bereits parallel ausgeführt wurde
                        i += 1
                    else:
                        current_data = extracted_data
                else:
                    current_data = extracted_data
            else:
                # Keine Extraktion, Daten bleiben unverändert (CF dient nur als Marker)
                pass

        elif isinstance(task, ParallelChain | ConditionalChain | ErrorHandlingChain):
            current_data = await task.a_run(current_data, **kwargs)

        elif callable(task) and not isinstance(task, (ChainBase, type)):
            # Check if the function is async, then await it
            if asyncio.iscoroutinefunction(task):
                current_data = await task(current_data)
            # Otherwise, run the synchronous function normally
            else:
                current_data = task(current_data)
        elif hasattr(task, 'a_run'):
            current_data = await task.a_run(current_data, **kwargs)
        elif isinstance(task, IS):
            # IS needs to be paired with >> to form a ConditionalChain
            next_task_for_cond = self.tasks[i + 1] if (i + 1) < len(self.tasks) else None
            if next_task_for_cond:
                # Form a conditional chain on the fly
                conditional_task = ConditionalChain(task, next_task_for_cond)
                # Check for a false branch defined with %
                next_next_task = self.tasks[i + 2] if (i + 2) < len(self.tasks) else None
                if isinstance(next_next_task, ConditionalChain) and next_next_task.false_branch:
                    conditional_task.false_branch = next_next_task.false_branch
                    i += 1  # also skip the false branch marker

                current_data = await conditional_task.a_run(current_data, **kwargs)
                i += 1  # Skip the next task as it's part of the conditional
            else:
                raise ValueError("IS condition must be followed by a task to execute.")

        i += 1  # Gehe zur nächsten Aufgabe

    return current_data
ChainBase

Abstract base class for all chain types, providing common operators.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
class ChainBase:
    """Abstract base class for all chain types, providing common operators."""

    def __rshift__(self, other: Any) -> 'Chain':
        """Implements the >> operator to chain tasks sequentially."""
        if isinstance(self, Chain):
            new_tasks = self.tasks + [other]
            return Chain._create_chain(new_tasks)
        return Chain._create_chain([self, other])

    def __add__(self, other: Any) -> 'ParallelChain':
        """Implements the + operator for parallel execution."""
        return ParallelChain([self, other])

    def __and__(self, other: Any) -> 'ParallelChain':
        """Implements the & operator, an alias for parallel execution."""
        return ParallelChain([self, other])

    def __or__(self, other: Any) -> 'ErrorHandlingChain':
        """Implements the | operator for defining a fallback/error handling path."""
        return ErrorHandlingChain(self, other)

    def __mod__(self, other: Any) -> 'ConditionalChain':
        """Implements the % operator for defining a false/else branch in a condition."""
        # This is typically used after a conditional chain.
        if isinstance(self, ConditionalChain):
            self.false_branch = other
            return self
        # Allows creating a conditional chain directly
        return ConditionalChain(None, self, other)

    def set_progress_callback(self, progress_tracker: 'ProgressTracker'):
        """Recursively sets the progress callback for all tasks in the chain."""
        tasks_to_process = []
        if hasattr(self, 'tasks'): tasks_to_process.extend(self.tasks)  # Chain
        if hasattr(self, 'agents'): tasks_to_process.extend(self.agents)  # ParallelChain
        if hasattr(self, 'true_branch'): tasks_to_process.append(self.true_branch)  # ConditionalChain
        if hasattr(self, 'false_branch') and self.false_branch: tasks_to_process.append(
            self.false_branch)  # ConditionalChain
        if hasattr(self, 'primary'): tasks_to_process.append(self.primary)  # ErrorHandlingChain
        if hasattr(self, 'fallback'): tasks_to_process.append(self.fallback)  # ErrorHandlingChain

        for task in tasks_to_process:
            if hasattr(task, 'set_progress_callback'):
                task.set_progress_callback(progress_tracker)

    def __call__(self, *args, **kwargs):
        """Allows the chain to be called like a function, returning an awaitable runner."""
        return self._Runner(self, args, kwargs)

    class _Runner:
        def __init__(self, parent, args, kwargs):
            self.parent = parent
            self.args = args
            self.kwargs = kwargs

        def __call__(self):
            """Synchronous execution."""
            return asyncio.run(self.parent.a_run(*self.args, **self.kwargs))

        def __await__(self):
            """Asynchronous execution."""
            return self.parent.a_run(*self.args, **self.kwargs).__await__()
__add__(other)

Implements the + operator for parallel execution.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
57
58
59
def __add__(self, other: Any) -> 'ParallelChain':
    """Implements the + operator for parallel execution."""
    return ParallelChain([self, other])
__and__(other)

Implements the & operator, an alias for parallel execution.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
61
62
63
def __and__(self, other: Any) -> 'ParallelChain':
    """Implements the & operator, an alias for parallel execution."""
    return ParallelChain([self, other])
__call__(*args, **kwargs)

Allows the chain to be called like a function, returning an awaitable runner.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
93
94
95
def __call__(self, *args, **kwargs):
    """Allows the chain to be called like a function, returning an awaitable runner."""
    return self._Runner(self, args, kwargs)
__mod__(other)

Implements the % operator for defining a false/else branch in a condition.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
69
70
71
72
73
74
75
76
def __mod__(self, other: Any) -> 'ConditionalChain':
    """Implements the % operator for defining a false/else branch in a condition."""
    # This is typically used after a conditional chain.
    if isinstance(self, ConditionalChain):
        self.false_branch = other
        return self
    # Allows creating a conditional chain directly
    return ConditionalChain(None, self, other)
__or__(other)

Implements the | operator for defining a fallback/error handling path.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
65
66
67
def __or__(self, other: Any) -> 'ErrorHandlingChain':
    """Implements the | operator for defining a fallback/error handling path."""
    return ErrorHandlingChain(self, other)
__rshift__(other)

Implements the >> operator to chain tasks sequentially.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
50
51
52
53
54
55
def __rshift__(self, other: Any) -> 'Chain':
    """Implements the >> operator to chain tasks sequentially."""
    if isinstance(self, Chain):
        new_tasks = self.tasks + [other]
        return Chain._create_chain(new_tasks)
    return Chain._create_chain([self, other])
set_progress_callback(progress_tracker)

Recursively sets the progress callback for all tasks in the chain.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
78
79
80
81
82
83
84
85
86
87
88
89
90
91
def set_progress_callback(self, progress_tracker: 'ProgressTracker'):
    """Recursively sets the progress callback for all tasks in the chain."""
    tasks_to_process = []
    if hasattr(self, 'tasks'): tasks_to_process.extend(self.tasks)  # Chain
    if hasattr(self, 'agents'): tasks_to_process.extend(self.agents)  # ParallelChain
    if hasattr(self, 'true_branch'): tasks_to_process.append(self.true_branch)  # ConditionalChain
    if hasattr(self, 'false_branch') and self.false_branch: tasks_to_process.append(
        self.false_branch)  # ConditionalChain
    if hasattr(self, 'primary'): tasks_to_process.append(self.primary)  # ErrorHandlingChain
    if hasattr(self, 'fallback'): tasks_to_process.append(self.fallback)  # ErrorHandlingChain

    for task in tasks_to_process:
        if hasattr(task, 'set_progress_callback'):
            task.set_progress_callback(progress_tracker)
ConditionalChain

Bases: ChainBase

Handles conditional execution based on a condition.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
class ConditionalChain(ChainBase):
    """Handles conditional execution based on a condition."""

    def __init__(self, condition: IS, true_branch: Any, false_branch: Any = None):
        self.condition = condition
        self.true_branch = true_branch
        self.false_branch = false_branch

    async def a_run(self, data: Any, **kwargs):
        """Executes the true or false branch based on the condition."""
        condition_met = False
        if isinstance(self.condition, IS) and isinstance(data, dict):
            if data.get(self.condition.key) == self.condition.expected_value:
                condition_met = True

        if condition_met:
            return await self.true_branch.a_run(data, **kwargs)
        elif self.false_branch:
            return await self.false_branch.a_run(data, **kwargs)
        return data  # Return original data if condition not met and no false branch
a_run(data, **kwargs) async

Executes the true or false branch based on the condition.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
160
161
162
163
164
165
166
167
168
169
170
171
async def a_run(self, data: Any, **kwargs):
    """Executes the true or false branch based on the condition."""
    condition_met = False
    if isinstance(self.condition, IS) and isinstance(data, dict):
        if data.get(self.condition.key) == self.condition.expected_value:
            condition_met = True

    if condition_met:
        return await self.true_branch.a_run(data, **kwargs)
    elif self.false_branch:
        return await self.false_branch.a_run(data, **kwargs)
    return data  # Return original data if condition not met and no false branch
ErrorHandlingChain

Bases: ChainBase

Handles exceptions in a primary chain by executing a fallback chain.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
174
175
176
177
178
179
180
181
182
183
184
185
186
187
class ErrorHandlingChain(ChainBase):
    """Handles exceptions in a primary chain by executing a fallback chain."""

    def __init__(self, primary: Any, fallback: Any):
        self.primary = primary
        self.fallback = fallback

    async def a_run(self, query: Any, **kwargs):
        """Tries the primary chain and executes the fallback on failure."""
        try:
            return await self.primary.a_run(query, **kwargs)
        except Exception as e:
            print(f"Primary chain failed with error: {e}. Running fallback.")
            return await self.fallback.a_run(query, **kwargs)
a_run(query, **kwargs) async

Tries the primary chain and executes the fallback on failure.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
181
182
183
184
185
186
187
async def a_run(self, query: Any, **kwargs):
    """Tries the primary chain and executes the fallback on failure."""
    try:
        return await self.primary.a_run(query, **kwargs)
    except Exception as e:
        print(f"Primary chain failed with error: {e}. Running fallback.")
        return await self.fallback.a_run(query, **kwargs)
Function

Bases: ChainBase

A wrapper to treat native Python functions as chainable components.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class Function(ChainBase):
    """A wrapper to treat native Python functions as chainable components."""

    def __init__(self, func: callable):
        if not callable(func):
            raise TypeError("Function object must be initialized with a callable.")
        self.func = func
        # Get a meaningful name for visualization
        self.func_name = getattr(func, '__name__', 'anonymous_lambda')

    async def a_run(self, data: Any, **kwargs):
        """Executes the wrapped function, handling both sync and async cases."""
        # Note: kwargs from the chain run are not passed to the native function
        # to maintain a simple, predictable (data in -> data out) interface.
        if asyncio.iscoroutinefunction(self.func):
            return await self.func(data)
        else:
            return self.func(data)

    def __repr__(self):
        return f"Function(name='{self.func_name}')"
a_run(data, **kwargs) async

Executes the wrapped function, handling both sync and async cases.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
121
122
123
124
125
126
127
128
async def a_run(self, data: Any, **kwargs):
    """Executes the wrapped function, handling both sync and async cases."""
    # Note: kwargs from the chain run are not passed to the native function
    # to maintain a simple, predictable (data in -> data out) interface.
    if asyncio.iscoroutinefunction(self.func):
        return await self.func(data)
    else:
        return self.func(data)
IS

Conditional check for branching logic.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
37
38
39
40
41
42
class IS:
    """Conditional check for branching logic."""

    def __init__(self, key: str, expected_value: Any = True):
        self.key = key
        self.expected_value = expected_value
ParallelChain

Bases: ChainBase

Handles parallel execution of multiple agents or chains.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
class ParallelChain(ChainBase):
    """Handles parallel execution of multiple agents or chains."""

    def __init__(self, agents: list[Union['FlowAgent', ChainBase]]):
        self.agents = agents

    async def a_run(self, query: Any, **kwargs):
        """Runs all agents/chains in parallel."""
        tasks = [agent.a_run(query, **kwargs) for agent in self.agents]
        results = await asyncio.gather(*tasks)
        return self._combine_results(results)

    def _combine_results(self, results: list[Any]) -> Any:
        """Intelligently combines parallel results."""
        if all(isinstance(r, str) for r in results):
            return " | ".join(results)
        return results
a_run(query, **kwargs) async

Runs all agents/chains in parallel.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
139
140
141
142
143
async def a_run(self, query: Any, **kwargs):
    """Runs all agents/chains in parallel."""
    tasks = [agent.a_run(query, **kwargs) for agent in self.agents]
    results = await asyncio.gather(*tasks)
    return self._combine_results(results)
chain_to_graph(self)

Convert chain to hierarchical structure with complete component detection.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
def chain_to_graph(self) -> dict[str, Any]:
    """Convert chain to hierarchical structure with complete component detection."""

    def process_component(comp, depth=0, visited=None):
        if visited is None:
            visited = set()

        # Prevent infinite recursion
        comp_id = id(comp)
        if comp_id in visited or depth > 20:
            return {"type": "Circular", "display": "[CIRCULAR_REF]", "depth": depth}
        visited.add(comp_id)

        if comp is None:
            return {"type": "Error", "display": "[NULL]", "depth": depth}

        try:
            # Agent detection
            if hasattr(comp, 'amd') and comp.amd:
                return {
                    "type": "Agent",
                    "display": f"[Agent] {comp.amd.name}",
                    "name": comp.amd.name,
                    "depth": depth
                }

            # Format detection (CF) with parallel detection
            if hasattr(comp, 'format_class'):
                name = comp.format_class.__name__
                display = f"[Format] {name}"

                result = {
                    "type": "Format",
                    "display": display,
                    "format_class": name,
                    "extract_key": getattr(comp, 'extract_key', None),
                    "depth": depth,
                    "creates_parallel": False
                }

                # Extract key visualization
                if hasattr(comp, 'extract_key') and comp.extract_key:
                    key = comp.extract_key
                    if key == '*':
                        display += " \033[90m(*all*)\033[0m"
                    elif isinstance(key, str):
                        display += f" \033[90m(→{key})\033[0m"
                    elif isinstance(key, tuple):
                        display += f" \033[90m(→{','.join(key)})\033[0m"

                # Parallel detection
                if hasattr(comp, 'parallel_count') and comp.parallel_count == 'n':
                    display += " \033[95m[PARALLEL]\033[0m"
                    result["creates_parallel"] = True
                    result["parallel_type"] = "auto_n"

                result["display"] = display
                return result

            # Condition detection (IS)
            if hasattr(comp, 'key') and hasattr(comp, 'expected_value'):
                return {
                    "type": "Condition",
                    "display": f"[Condition] IS {comp.key}=='{comp.expected_value}'",
                    "condition_key": comp.key,
                    "expected_value": comp.expected_value,
                    "depth": depth
                }

            # Parallel chain detection
            if hasattr(comp, 'agents') and isinstance(comp.agents, list | tuple):
                branches = []
                for i, agent in enumerate(comp.agents):
                    if agent:
                        branch_data = process_component(agent, depth + 1, visited.copy())
                        branch_data["branch_id"] = i
                        branches.append(branch_data)

                return {
                    "type": "Parallel",
                    "display": f"[Parallel] {len(branches)} branches",
                    "branches": branches,
                    "branch_count": len(branches),
                    "execution_type": "concurrent",
                    "depth": depth
                }

            if isinstance(comp, Function):
                return {
                    "type": "Function",
                    "display": f"[Func] {comp.func_name}",
                    "function_name": comp.func_name,
                    "depth": depth
                }

            # Conditional chain detection
            if hasattr(comp, 'condition') and hasattr(comp, 'true_branch'):
                condition_data = process_component(comp.condition, depth + 1,
                                                   visited.copy()) if comp.condition else None
                true_data = process_component(comp.true_branch, depth + 1, visited.copy()) if comp.true_branch else None
                false_data = None

                if hasattr(comp, 'false_branch') and comp.false_branch:
                    false_data = process_component(comp.false_branch, depth + 1, visited.copy())

                return {
                    "type": "Conditional",
                    "display": "[Conditional] Branch Logic",
                    "condition": condition_data,
                    "true_branch": true_data,
                    "false_branch": false_data,
                    "has_false_branch": false_data is not None,
                    "depth": depth
                }

            # Error handling detection
            if hasattr(comp, 'primary') and hasattr(comp, 'fallback'):
                primary_data = process_component(comp.primary, depth + 1, visited.copy()) if comp.primary else None
                fallback_data = process_component(comp.fallback, depth + 1, visited.copy()) if comp.fallback else None

                return {
                    "type": "ErrorHandling",
                    "display": "[Try-Catch] Error Handler",
                    "primary": primary_data,
                    "fallback": fallback_data,
                    "has_fallback": fallback_data is not None,
                    "depth": depth
                }

            # Regular chain detection
            if hasattr(comp, 'tasks') and isinstance(comp.tasks, list | tuple):
                tasks = []
                for i, task in enumerate(comp.tasks):
                    if task is not None:
                        task_data = process_component(task, depth + 1, visited.copy())
                        task_data["task_id"] = i
                        tasks.append(task_data)

                # Analyze chain characteristics
                has_conditionals = any(t.get("type") == "Conditional" for t in tasks)
                has_parallels = any(t.get("type") == "Parallel" for t in tasks)
                has_error_handling = any(t.get("type") == "ErrorHandling" for t in tasks)
                has_auto_parallel = any(t.get("creates_parallel", False) for t in tasks)

                chain_type = "Sequential"
                if has_auto_parallel:
                    chain_type = "Auto-Parallel"
                elif has_conditionals and has_parallels:
                    chain_type = "Complex"
                elif has_conditionals:
                    chain_type = "Conditional"
                elif has_parallels:
                    chain_type = "Mixed-Parallel"
                elif has_error_handling:
                    chain_type = "Error-Handling"

                return {
                    "type": "Chain",
                    "display": f"[Chain] {chain_type}",
                    "tasks": tasks,
                    "task_count": len(tasks),
                    "chain_type": chain_type,
                    "has_conditionals": has_conditionals,
                    "has_parallels": has_parallels,
                    "has_error_handling": has_error_handling,
                    "has_auto_parallel": has_auto_parallel,
                    "depth": depth
                }

            # Fallback for unknown types
            return {
                "type": "Unknown",
                "display": f"[Unknown] {type(comp).__name__}",
                "class_name": type(comp).__name__,
                "depth": depth
            }

        except Exception as e:
            return {
                "type": "Error",
                "display": f"[ERROR] {str(e)[:50]}",
                "error": str(e),
                "depth": depth
            }
        finally:
            visited.discard(comp_id)

    return {"structure": process_component(self)}
print_graph(self)

Enhanced chain visualization with complete functionality coverage and parallel detection.

Source code in toolboxv2/mods/isaa/base/Agent/chain.py
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
def print_graph(self):
    """Enhanced chain visualization with complete functionality coverage and parallel detection."""

    # Enhanced color scheme with parallel indicators
    COLORS = {
        "Agent": "\033[94m",  # Blue
        "Format": "\033[92m",  # Green
        "Condition": "\033[93m",  # Yellow
        "Parallel": "\033[95m",  # Magenta
        "Function": "\033[35m",  # Light Purple
        "Conditional": "\033[96m",  # Cyan
        "ErrorHandling": "\033[91m",  # Red
        "Chain": "\033[97m",  # White
        "Unknown": "\033[31m",  # Dark Red
        "Error": "\033[91m",  # Red
        "AutoParallel": "\033[105m",  # Bright Magenta Background
    }
    RESET = "\033[0m"
    BOLD = "\033[1m"
    DIM = "\033[2m"
    PARALLEL_ICON = "⚡"
    BRANCH_ICON = "🔀"
    ERROR_ICON = "🚨"
    FUNCTION_ICON = "ƒ"

    def style_component(comp, override_color=None):
        """Apply enhanced styling with parallel indicators."""
        if not comp:
            return f"{COLORS['Error']}[NULL]{RESET}"

        comp_type = comp.get("type", "Unknown")
        display = comp.get("display", f"[{comp_type}]")
        color = override_color or COLORS.get(comp_type, COLORS['Unknown'])
        # Special handling for parallel-creating formats
        if comp_type == "Format" and comp.get("creates_parallel", False):
            return f"{color}{PARALLEL_ICON} {display}{RESET}"
        elif comp_type == "Function":
            return f"{color}{FUNCTION_ICON} {display}{RESET}"
        else:
            color = override_color or COLORS.get(comp_type, COLORS['Unknown'])
            return f"{color}{display}{RESET}"

    def print_section_header(title, details=None):
        """Print formatted section header."""
        print(f"\n{BOLD}{'=' * 60}{RESET}")
        print(f"{BOLD}🔗 {title}{RESET}")
        if details:
            print(f"{DIM}{details}{RESET}")
        print(f"{BOLD}{'=' * 60}{RESET}")

    def render_task_flow(tasks, indent="", show_parallel_creation=True):
        """Render tasks with parallel creation detection."""
        if not tasks:
            print(f"{indent}{DIM}(No tasks){RESET}")
            return

        for i, task in enumerate(tasks):
            if not task:
                continue

            is_last = i == len(tasks) - 1
            connector = "└─ " if is_last else "├─ "
            next_indent = indent + ("    " if is_last else "│   ")

            task_type = task.get("type", "Unknown")

            # Handle different task types
            if task_type == "Format" and task.get("creates_parallel", False):
                print(f"{indent}{connector}{style_component(task)}")

                # Show what happens next
                if i + 1 < len(tasks):
                    next_task = tasks[i + 1]
                    print(f"{next_indent}├─ {DIM}Creates parallel execution for:{RESET}")
                    print(f"{next_indent}└─ {PARALLEL_ICON} {style_component(next_task)}")
                    # Skip the next task in main loop since we showed it here
                    continue

            elif task_type == "Parallel":
                print(f"{indent}{connector}{style_component(task)}")
                branches = task.get("branches", [])

                for j, branch in enumerate(branches):
                    if branch:
                        branch_last = j == len(branches) - 1
                        branch_conn = "└─ " if branch_last else "├─ "
                        branch_indent = next_indent + ("    " if branch_last else "│   ")

                        print(f"{next_indent}{branch_conn}{BRANCH_ICON} Branch {j + 1}:")

                        if branch.get("type") == "Chain":
                            render_task_flow(branch.get("tasks", []), branch_indent, False)
                        else:
                            print(f"{branch_indent}└─ {style_component(branch)}")

            elif task_type == "Conditional":
                print(f"{indent}{connector}{style_component(task)}")

                # Condition
                condition = task.get("condition")
                if condition:
                    print(f"{next_indent}├─ {style_component(condition)}")

                # True branch
                true_branch = task.get("true_branch")
                false_branch = task.get("false_branch")
                has_false = false_branch is not None

                if true_branch:
                    true_conn = "├─ " if has_false else "└─ "
                    print(f"{next_indent}{true_conn}{COLORS['Conditional']}✓ TRUE:{RESET}")
                    true_indent = next_indent + ("│   " if has_false else "    ")

                    if true_branch.get("type") == "Chain":
                        render_task_flow(true_branch.get("tasks", []), true_indent, False)
                    else:
                        print(f"{true_indent}└─ {style_component(true_branch)}")

                if false_branch:
                    print(f"{next_indent}└─ {COLORS['Conditional']}✗ FALSE:{RESET}")
                    false_indent = next_indent + "    "

                    if false_branch.get("type") == "Chain":
                        render_task_flow(false_branch.get("tasks", []), false_indent, False)
                    else:
                        print(f"{false_indent}└─ {style_component(false_branch)}")

            elif task_type == "ErrorHandling":
                print(f"{indent}{connector}{style_component(task)}")

                primary = task.get("primary")
                fallback = task.get("fallback")
                has_fallback = fallback is not None

                if primary:
                    prim_conn = "├─ " if has_fallback else "└─ "
                    print(f"{next_indent}{prim_conn}{COLORS['Chain']}🎯 PRIMARY:{RESET}")
                    prim_indent = next_indent + ("│   " if has_fallback else "    ")

                    if primary.get("type") == "Chain":
                        render_task_flow(primary.get("tasks", []), prim_indent, False)
                    else:
                        print(f"{prim_indent}└─ {style_component(primary)}")

                if fallback:
                    print(f"{next_indent}└─ {ERROR_ICON} FALLBACK:")
                    fallback_indent = next_indent + "    "

                    if fallback.get("type") == "Chain":
                        render_task_flow(fallback.get("tasks", []), fallback_indent, False)
                    else:
                        print(f"{fallback_indent}└─ {style_component(fallback)}")

            else:
                print(f"{indent}{connector}{style_component(task)}")

    # Main execution
    try:
        # Generate graph structure
        graph_data = self.chain_to_graph()
        structure = graph_data.get("structure")

        if not structure:
            print_section_header("Empty Chain")
            return

        # Determine chain characteristics
        chain_type = structure.get("chain_type", "Unknown")
        has_auto_parallel = structure.get("has_auto_parallel", False)
        has_parallels = structure.get("has_parallels", False)
        has_conditionals = structure.get("has_conditionals", False)
        has_error_handling = structure.get("has_error_handling", False)
        task_count = structure.get("task_count", 0)

        # Build header info
        info_parts = [f"Tasks: {task_count}"]
        if has_auto_parallel:
            info_parts.append(f"{PARALLEL_ICON} Auto-Parallel")
        if has_parallels:
            info_parts.append(f"{BRANCH_ICON} Parallel Branches")
        if has_conditionals:
            info_parts.append("🔀 Conditionals")
        if has_error_handling:
            info_parts.append(f"{ERROR_ICON} Error Handling")

        print_section_header(f"Chain Visualization - {chain_type}", " | ".join(info_parts))

        # Handle different structure types
        struct_type = structure.get("type", "Unknown")

        if struct_type == "Chain":
            tasks = structure.get("tasks", [])
            render_task_flow(tasks)

        elif struct_type == "Parallel":
            print(f"{style_component(structure)}")
            branches = structure.get("branches", [])
            for i, branch in enumerate(branches):
                is_last = i == len(branches) - 1
                conn = "└─ " if is_last else "├─ "
                indent = "    " if is_last else "│   "

                print(f"{conn}{BRANCH_ICON} Branch {i + 1}:")
                if branch.get("type") == "Chain":
                    render_task_flow(branch.get("tasks", []), indent, False)
                else:
                    print(f"{indent}└─ {style_component(branch)}")

        elif struct_type == "Conditional" or struct_type == "ErrorHandling":
            render_task_flow([structure])

        else:
            print(f"└─ {style_component(structure)}")

        print(f"\n{DIM}{'─' * 60}{RESET}")

    except Exception as e:
        print(f"\n{COLORS['Error']}{BOLD}[VISUALIZATION ERROR]{RESET}")
        print(f"{COLORS['Error']}Error: {str(e)}{RESET}")

        # Emergency fallback
        print(f"\n{DIM}--- Emergency Info ---{RESET}")
        try:
            attrs = []
            for attr in ['tasks', 'agents', 'condition', 'true_branch', 'false_branch', 'primary', 'fallback']:
                if hasattr(self, attr):
                    val = getattr(self, attr)
                    if val is not None:
                        if isinstance(val, list | tuple):
                            attrs.append(f"{attr}: {len(val)} items")
                        else:
                            attrs.append(f"{attr}: {type(val).__name__}")

            if attrs:
                print("Chain attributes:")
                for attr in attrs:
                    print(f"  • {attr}")
        except:
            print("Complete inspection failed")

        print(f"{DIM}--- End Emergency Info ---{RESET}\n")
config
A2AConfig

Bases: BaseModel

Configuration for A2A integration.

Source code in toolboxv2/mods/isaa/base/Agent/config.py
116
117
118
119
120
121
122
class A2AConfig(BaseModel):
    """Configuration for A2A integration."""
    server: dict[str, Any] | None = Field(default=None, description="Configuration to run an A2A server (host, port, etc.).")
    known_agents: dict[str, str] = Field(default_factory=dict, description="Named A2A agent URLs to interact with (e.g., {'weather_agent': 'http://weather:5000'}).")
    default_task_timeout: int = Field(default=120, description="Default timeout in seconds for waiting on A2A task results.")

    model_config = ConfigDict(arbitrary_types_allowed=True)
ADKConfig

Bases: BaseModel

Configuration for ADK integration.

Source code in toolboxv2/mods/isaa/base/Agent/config.py
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
class ADKConfig(BaseModel):
    """Configuration for ADK integration."""
    enabled: bool = Field(default=True, description="Enable ADK features if ADK is installed.")
    description: str | None = Field(default=None, description="ADK LlmAgent description.")
    instruction_override: str | None = Field(default=None, description="Override agent's system message for ADK.")
    # Tools added via builder or auto-discovery
    code_executor: str | BaseCodeExecutor | None = Field(default=None, description="Reference name or instance of ADK code executor.")
    planner: str | BasePlanner | None = Field(default=None, description="Reference name or instance of ADK planner.")
    examples: list[Example] | None = Field(default=None, description="Few-shot examples for ADK.")
    output_schema: type[BaseModel] | None = Field(default=None, description="Pydantic model for structured output.")
    # MCP Toolset config handled separately if ADK is enabled
    use_mcp_toolset: bool = Field(default=True, description="Use ADK's MCPToolset for MCP client connections if ADK is enabled.")
    # Runner config handled separately

    model_config = ConfigDict(arbitrary_types_allowed=True)
AgentConfig

Bases: BaseModel

Main configuration schema for an EnhancedAgent.

Source code in toolboxv2/mods/isaa/base/Agent/config.py
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
class AgentConfig(BaseModel):
    """Main configuration schema for an EnhancedAgent."""
    agent_name: str = Field(..., description="Unique name for this agent instance.")
    version: str = Field(default="0.1.0")

    agent_instruction: str = Field(default="You are a helpful AI assistant. Answer user questions to the best of your knowledge. Respond concisely. use tools when needed")
    agent_description: str = Field(default="An configurable, production-ready agent with integrated capabilities.")

    # Model Selection
    models: list[ModelConfig] = Field(..., description="List of available LLM configurations.")
    default_llm_model: str = Field(..., description="Name of the ModelConfig to use for general LLM calls.")
    formatter_llm_model: str | None = Field(default=None, description="Optional: Name of a faster/cheaper ModelConfig for a_format_class calls.")

    # Core Agent Settings
    world_model_initial_data: dict[str, Any] | None = Field(default=None)
    enable_streaming: bool = Field(default=False)
    verbose: bool = Field(default=False)
    log_level: str = Field(default="INFO", description="Logging level (DEBUG, INFO, WARNING, ERROR).")
    max_history_length: int = Field(default=20, description="Max conversation turns for LiteLLM history.")
    trim_strategy: Literal["litellm", "basic"] = Field(default="litellm")
    persist_history: bool = Field(default=True, description="Persist conversation history (requires persistent ChatSession).")
    user_id_default: str | None = Field(default=None, description="Default user ID for interactions.")

    # Secure Code Execution
    code_executor_type: Literal["restricted", "docker", "none"] = Field(default="restricted", description="Type of code executor to use.")
    code_executor_config: dict[str, Any] = Field(default_factory=dict, description="Configuration specific to the chosen code executor.")
    enable_adk_code_execution_tool: bool = Field(default=True, description="Expose code execution as an ADK tool if ADK is enabled.")

    # Framework Integrations
    adk: ADKConfig | None = Field(default_factory=ADKConfig if ADK_AVAILABLE_CONF else lambda: None)
    mcp: MCPConfig | None = Field(default_factory=MCPConfig if MCP_AVAILABLE_CONF else lambda: None)
    a2a: A2AConfig | None = Field(default_factory=A2AConfig if A2A_AVAILABLE_CONF else lambda: None)

    # Observability & Cost
    observability: ObservabilityConfig | None = Field(default_factory=ObservabilityConfig)
    budget_manager: BudgetManager | None = Field(default=None, description="Global LiteLLM budget manager instance.") # Needs to be passed in

    # Human-in-the-Loop
    enable_hitl: bool = Field(default=False, description="Enable basic Human-in-the-Loop hooks.")

    # Add other global settings as needed

    model_config = ConfigDict(arbitrary_types_allowed=True)

    @model_validator(mode='after')
    def validate_model_references(self) -> 'AgentConfig':
        model_names = {m.name for m in self.models}
        if self.default_llm_model not in model_names:
            raise ValueError(f"default_llm_model '{self.default_llm_model}' not found in defined models.")
        if self.formatter_llm_model and self.formatter_llm_model not in model_names:
            raise ValueError(f"formatter_llm_model '{self.formatter_llm_model}' not found in defined models.")
        return self

    @model_validator(mode='after')
    def validate_framework_availability(self) -> 'AgentConfig':
        if self.adk and self.adk.enabled and not ADK_AVAILABLE_CONF:
            logger.warning("ADK configuration provided but ADK library not installed. Disabling ADK features.")
            self.adk.enabled = False
        if self.mcp and (self.mcp.server or self.mcp.client_connections) and not MCP_AVAILABLE_CONF:
             logger.warning("MCP configuration provided but MCP library not installed. Disabling MCP features.")
             self.mcp = None # Or disable specific parts
        if self.a2a and (self.a2a.server or self.a2a.known_agents) and not A2A_AVAILABLE_CONF:
             logger.warning("A2A configuration provided but A2A library not installed. Disabling A2A features.")
             self.a2a = None # Or disable specific parts
        return self

    @classmethod
    def load_from_yaml(cls, path: str | Path) -> 'AgentConfig':
        """Loads configuration from a YAML file."""
        file_path = Path(path)
        if not file_path.is_file():
            raise FileNotFoundError(f"Configuration file not found: {path}")
        with open(file_path) as f:
            config_data = yaml.safe_load(f)
        logger.info(f"Loaded agent configuration from {path}")
        return cls(**config_data)

    def save_to_yaml(self, path: str | Path):
        """Saves the current configuration to a YAML file."""
        file_path = Path(path)
        file_path.parent.mkdir(parents=True, exist_ok=True)
        with open(file_path, 'w') as f:
            # Use Pydantic's model_dump for clean serialization
            yaml.dump(self.model_dump(mode='python'), f, sort_keys=False)
        logger.info(f"Saved agent configuration to {path}")
load_from_yaml(path) classmethod

Loads configuration from a YAML file.

Source code in toolboxv2/mods/isaa/base/Agent/config.py
201
202
203
204
205
206
207
208
209
210
@classmethod
def load_from_yaml(cls, path: str | Path) -> 'AgentConfig':
    """Loads configuration from a YAML file."""
    file_path = Path(path)
    if not file_path.is_file():
        raise FileNotFoundError(f"Configuration file not found: {path}")
    with open(file_path) as f:
        config_data = yaml.safe_load(f)
    logger.info(f"Loaded agent configuration from {path}")
    return cls(**config_data)
save_to_yaml(path)

Saves the current configuration to a YAML file.

Source code in toolboxv2/mods/isaa/base/Agent/config.py
212
213
214
215
216
217
218
219
def save_to_yaml(self, path: str | Path):
    """Saves the current configuration to a YAML file."""
    file_path = Path(path)
    file_path.parent.mkdir(parents=True, exist_ok=True)
    with open(file_path, 'w') as f:
        # Use Pydantic's model_dump for clean serialization
        yaml.dump(self.model_dump(mode='python'), f, sort_keys=False)
    logger.info(f"Saved agent configuration to {path}")
MCPConfig

Bases: BaseModel

Configuration for MCP integration.

Source code in toolboxv2/mods/isaa/base/Agent/config.py
107
108
109
110
111
112
113
class MCPConfig(BaseModel):
    """Configuration for MCP integration."""
    server: dict[str, Any] | None = Field(default=None, description="Configuration to run an MCP server (host, port, etc.).")
    client_connections: dict[str, str] = Field(default_factory=dict, description="Named MCP server URLs to connect to as a client (e.g., {'files': 'stdio:npx @mcp/server-filesystem /data'}).")
    # ADK's MCPToolset handles client connections if ADKConfig.use_mcp_toolset is True

    model_config = ConfigDict(arbitrary_types_allowed=True)
ModelConfig

Bases: BaseModel

Configuration specific to an LLM model via LiteLLM.

Source code in toolboxv2/mods/isaa/base/Agent/config.py
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
class ModelConfig(BaseModel):
    """Configuration specific to an LLM model via LiteLLM."""
    # Used as key for model selection
    name: str = Field(..., description="Unique identifier/alias for this model configuration (e.g., 'fast_formatter', 'main_reasoner').")
    model: str = Field(..., description="LiteLLM model string (e.g., 'gemini/gemini-1.5-pro-latest', 'ollama/mistral').")
    provider: str | None = Field(default=None, description="LiteLLM provider override if needed.")
    api_key: str | None = Field(default=None, description="API Key (consider using environment variables).")
    api_base: str | None = Field(default=None, description="API Base URL (for local models, proxies).")
    api_version: str | None = Field(default=None, description="API Version (e.g., for Azure).")

    # Common LLM Parameters
    temperature: float | None = Field(default=0.7)
    top_p: float | None = Field(default=None)
    top_k: int | None = Field(default=None)
    max_tokens: int | None = Field(default=2048, description="Max tokens for generation.")
    max_input_tokens: int | None = Field(default=None, description="Max input context window (autodetected if None).")
    stop_sequence: list[str] | None = Field(default=None)
    presence_penalty: float | None = Field(default=None)
    frequency_penalty: float | None = Field(default=None)
    system_message: str | None = Field(default=None, description="Default system message for this model.")

    # LiteLLM Specific
    caching: bool = Field(default=True, description="Enable LiteLLM caching for this model.")
    # budget_manager: Optional[BudgetManager] = Field(default=None) # Budget manager applied globally or per-agent

    model_config = ConfigDict(arbitrary_types_allowed=True, extra='allow') # Allow extra LiteLLM params
ObservabilityConfig

Bases: BaseModel

Configuration for observability (OpenTelemetry).

Source code in toolboxv2/mods/isaa/base/Agent/config.py
125
126
127
128
129
130
131
132
class ObservabilityConfig(BaseModel):
    """Configuration for observability (OpenTelemetry)."""
    enabled: bool = Field(default=True)
    endpoint: str | None = Field(default=None, description="OTLP endpoint URL (e.g., http://jaeger:4317).")
    service_name: str | None = Field(default=None, description="Service name for traces/metrics (defaults to agent name).")
    # Add more OTel config options as needed (headers, certs, resource attributes)

    model_config = ConfigDict(arbitrary_types_allowed=True)
executors
DockerCodeExecutor

Bases: _BaseExecutorClass

Executes Python code in a sandboxed Docker container.

Requires Docker to be installed and running, and the 'docker' Python SDK.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
class DockerCodeExecutor(_BaseExecutorClass):
    """
    Executes Python code in a sandboxed Docker container.

    Requires Docker to be installed and running, and the 'docker' Python SDK.
    """
    DEFAULT_DOCKER_IMAGE = "python:3.10-slim" # Use a minimal image
    DEFAULT_TIMEOUT = 10 # Seconds
    DEFAULT_MEM_LIMIT = "128m"
    DEFAULT_CPUS = 0.5

    def __init__(self,
                 docker_image: str = DEFAULT_DOCKER_IMAGE,
                 timeout: int = DEFAULT_TIMEOUT,
                 mem_limit: str = DEFAULT_MEM_LIMIT,
                 cpus: float = DEFAULT_CPUS,
                 network_mode: str = "none", # Disable networking by default for security
                 docker_client_config: dict | None = None):
        if not DOCKER_AVAILABLE:
            raise ImportError("Docker SDK not installed ('pip install docker'). Cannot use DockerCodeExecutor.")

        self.docker_image = docker_image
        self.timeout = timeout
        self.mem_limit = mem_limit
        self.cpus = cpus
        self.network_mode = network_mode
        try:
            self.client = docker.from_env(**(docker_client_config or {}))
            self.client.ping() # Check connection
            # Ensure image exists locally or pull it
            try:
                self.client.images.get(self.docker_image)
                logger.info(f"Docker image '{self.docker_image}' found locally.")
            except ImageNotFound:
                logger.warning(f"Docker image '{self.docker_image}' not found locally. Attempting to pull...")
                try:
                    self.client.images.pull(self.docker_image)
                    logger.info(f"Successfully pulled Docker image '{self.docker_image}'.")
                except APIError as pull_err:
                    raise RuntimeError(f"Failed to pull Docker image '{self.docker_image}': {pull_err}") from pull_err
        except Exception as e:
            raise RuntimeError(f"Failed to connect to Docker daemon: {e}. Is Docker running?") from e
        logger.info(f"DockerCodeExecutor initialized (Image: {docker_image}, Timeout: {timeout}s, Network: {network_mode})")

    def _execute(self, code: str) -> dict[str, Any]:
        """Internal execution logic."""
        result = {"stdout": "", "stderr": "", "error": None, "exit_code": None}
        container = None

        try:
            logger.debug(f"Creating Docker container from image '{self.docker_image}'...")
            container = self.client.containers.run(
                image=self.docker_image,
                command=["python", "-c", code],
                detach=True,
                mem_limit=self.mem_limit,
                nano_cpus=int(self.cpus * 1e9),
                network_mode=self.network_mode,
                # Security considerations: Consider read-only filesystem, dropping capabilities
                read_only=True,
                # working_dir="/app", # Define a working dir if needed
                # volumes={...} # Mount volumes carefully if required
            )
            logger.debug(f"Container '{container.short_id}' started.")

            # Wait for container completion with timeout
            container_result = container.wait(timeout=self.timeout)
            result["exit_code"] = container_result.get("StatusCode", None)

            # Retrieve logs
            result["stdout"] = container.logs(stdout=True, stderr=False).decode('utf-8', errors='replace').strip()
            result["stderr"] = container.logs(stdout=False, stderr=True).decode('utf-8', errors='replace').strip()

            logger.debug(f"Container '{container.short_id}' finished with exit code {result['exit_code']}.")
            if result["exit_code"] != 0:
                 logger.warning(f"Container stderr: {result['stderr'][:500]}...") # Log stderr on failure

        except ContainerError as e:
            result["error"] = f"ContainerError: {e}"
            result["stderr"] = e.stderr.decode('utf-8', errors='replace').strip() if e.stderr else str(e)
            result["exit_code"] = e.exit_status
            logger.error(f"Container '{container.short_id if container else 'N/A'}' failed: {result['error']}\nStderr: {result['stderr']}")
        except APIError as e:
            result["error"] = f"Docker APIError: {e}"
            result["exit_code"] = -1
            logger.error(f"Docker API error during execution: {e}")
        except Exception as e:
            # Catch potential timeout errors from container.wait or other unexpected issues
            result["error"] = f"Unexpected execution error: {type(e).__name__}: {e}"
            result["exit_code"] = -1
            # Check if it looks like a timeout
            if isinstance(e, TimeoutError) or "Timeout" in str(e): # docker SDK might raise requests.exceptions.ReadTimeout
                result["stderr"] = f"Execution timed out after {self.timeout} seconds."
                logger.warning(f"Container execution timed out ({self.timeout}s).")
            else:
                logger.error(f"Unexpected error during Docker execution: {e}", exc_info=True)
        finally:
            if container:
                try:
                    logger.debug(f"Removing container '{container.short_id}'...")
                    container.remove(force=True)
                except APIError as rm_err:
                    logger.warning(f"Failed to remove container {container.short_id}: {rm_err}")

        return result

     # --- ADK Compatibility Method ---
    if ADK_EXEC_AVAILABLE:
        def execute_code(self, invocation_context: InvocationContext, code_input: CodeExecutionInput) -> CodeExecutionResult:
            logger.debug(f"DockerCodeExecutor executing ADK request (lang: {code_input.language}). Code: {code_input.code[:100]}...")
            if code_input.language.lower() != 'python':
                 return CodeExecutionResult(output=f"Error: Unsupported language '{code_input.language}'. Only Python is supported.", outcome="OUTCOME_FAILURE")

            exec_result = self._execute(code_input.code)

            output_str = ""
            if exec_result["stdout"]:
                output_str += f"Stdout:\n{exec_result['stdout']}\n"
            if exec_result["stderr"]:
                 output_str += f"Stderr:\n{exec_result['stderr']}\n"
            if not output_str and exec_result["exit_code"] == 0:
                 output_str = "Execution successful with no output."
            elif not output_str and exec_result["exit_code"] != 0:
                 output_str = f"Execution failed with no output (Exit code: {exec_result['exit_code']}). Error: {exec_result['error']}"

            outcome = "OUTCOME_SUCCESS" if exec_result["exit_code"] == 0 else "OUTCOME_FAILURE"

            return CodeExecutionResult(output=output_str.strip(), outcome=outcome)
    # --- End ADK Compatibility ---

    # --- Direct Call Method ---
    def execute(self, code: str) -> dict[str, Any]:
        """Directly execute code, returning detailed dictionary."""
        logger.debug(f"DockerCodeExecutor executing direct call. Code: {code[:100]}...")
        return self._execute(code)
execute(code)

Directly execute code, returning detailed dictionary.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py
333
334
335
336
def execute(self, code: str) -> dict[str, Any]:
    """Directly execute code, returning detailed dictionary."""
    logger.debug(f"DockerCodeExecutor executing direct call. Code: {code[:100]}...")
    return self._execute(code)
RestrictedPythonExecutor

Bases: _BaseExecutorClass

Executes Python code using restrictedpython.

Safer than exec() but NOT a full sandbox. Known vulnerabilities exist. Use with extreme caution and only with trusted code sources or for low-risk operations. Docker is strongly recommended for untrusted code.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
class RestrictedPythonExecutor(_BaseExecutorClass):
    """
    Executes Python code using restrictedpython.

    Safer than exec() but NOT a full sandbox. Known vulnerabilities exist.
    Use with extreme caution and only with trusted code sources or for
    low-risk operations. Docker is strongly recommended for untrusted code.
    """
    DEFAULT_ALLOWED_GLOBALS = {
        **safe_globals,
        '_print_': restrictedpython.PrintCollector,
        '_getattr_': restrictedpython.safe_getattr,
        '_getitem_': restrictedpython.safe_getitem,
        '_write_': restrictedpython.guarded_setattr, # Allows modifying specific safe objects if needed
        # Add other safe builtins or modules carefully
        'math': __import__('math'),
        'random': __import__('random'),
        'datetime': __import__('datetime'),
        'time': __import__('time'),
        # 'requests': None, # Example: Explicitly disallow
    }

    def __init__(self, allowed_globals: dict | None = None, max_execution_time: int = 5):
        if not RESTRICTEDPYTHON_AVAILABLE:
            raise ImportError("restrictedpython is not installed. Cannot use RestrictedPythonExecutor.")
        self.allowed_globals = allowed_globals or self.DEFAULT_ALLOWED_GLOBALS
        self.max_execution_time = max_execution_time # Basic timeout (not perfectly enforced by restrictedpython)
        logger.warning("Initialized RestrictedPythonExecutor. This provides LIMITED sandboxing. Use Docker for untrusted code.")

    def _execute(self, code: str) -> dict[str, Any]:
        """Internal execution logic."""
        start_time = time.monotonic()
        result = {"stdout": "", "stderr": "", "error": None, "exit_code": None}
        local_vars = {}
        stdout_capture = io.StringIO()
        stderr_capture = io.StringIO()

        try:
            # Basic timeout check (not preemptive)
            if time.monotonic() - start_time > self.max_execution_time:
                 raise TimeoutError(f"Execution exceeded max time of {self.max_execution_time}s (pre-check).")

            # Compile the code in restricted mode
            byte_code = compile_restricted(code, filename='<inline code>', mode='exec')

            # Add a print collector to capture output
            self.allowed_globals['_print_'] = restrictedpython.PrintCollector
            print_collector = self.allowed_globals['_print_']()
            exec_globals = {**self.allowed_globals, '_print': print_collector}

            # Execute the compiled code
            # Note: restrictedpython does not inherently support robust timeouts during exec
            exec(byte_code, exec_globals, local_vars)

            # Check execution time again
            duration = time.monotonic() - start_time
            if duration > self.max_execution_time:
                logger.warning(f"Execution finished but exceeded max time ({duration:.2f}s > {self.max_execution_time}s).")
                # Potentially treat as an error or partial success

            result["stdout"] = print_collector.printed_text # Access collected prints
            result["exit_code"] = 0 # Assume success if no exception

        except TimeoutError as e:
            result["stderr"] = f"TimeoutError: {e}"
            result["error"] = str(e)
            result["exit_code"] = -1 # Indicate timeout
        except SyntaxError as e:
            result["stderr"] = f"SyntaxError: {e}"
            result["error"] = str(e)
            result["exit_code"] = 1
        except Exception as e:
            # Capture other potential execution errors allowed by restrictedpython
            error_type = type(e).__name__
            error_msg = f"{error_type}: {e}"
            result["stderr"] = error_msg
            result["error"] = str(e)
            result["exit_code"] = 1
            logger.warning(f"RestrictedPython execution caught exception: {error_msg}", exc_info=False) # Avoid logging potentially sensitive details from code
        finally:
            stdout_capture.close() # Not used directly with PrintCollector
            stderr_capture.close()

        return result

    # --- ADK Compatibility Method ---
    if ADK_EXEC_AVAILABLE:
        def execute_code(self, invocation_context: InvocationContext, code_input: CodeExecutionInput) -> CodeExecutionResult:
            logger.debug(f"RestrictedPythonExecutor executing ADK request (lang: {code_input.language}). Code: {code_input.code[:100]}...")
            if code_input.language.lower() != 'python':
                 return CodeExecutionResult(output=f"Error: Unsupported language '{code_input.language}'. Only Python is supported.", outcome="OUTCOME_FAILURE")

            exec_result = self._execute(code_input.code)

            output_str = ""
            if exec_result["stdout"]:
                output_str += f"Stdout:\n{exec_result['stdout']}\n"
            if exec_result["stderr"]:
                 output_str += f"Stderr:\n{exec_result['stderr']}\n"
            if not output_str and exec_result["exit_code"] == 0:
                 output_str = "Execution successful with no output."
            elif not output_str and exec_result["exit_code"] != 0:
                 output_str = f"Execution failed with no output (Exit code: {exec_result['exit_code']}). Error: {exec_result['error']}"


            outcome = "OUTCOME_SUCCESS" if exec_result["exit_code"] == 0 else "OUTCOME_FAILURE"

            return CodeExecutionResult(output=output_str.strip(), outcome=outcome)
    # --- End ADK Compatibility ---

    # --- Direct Call Method ---
    def execute(self, code: str) -> dict[str, Any]:
        """Directly execute code, returning detailed dictionary."""
        logger.debug(f"RestrictedPythonExecutor executing direct call. Code: {code[:100]}...")
        return self._execute(code)
execute(code)

Directly execute code, returning detailed dictionary.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py
193
194
195
196
def execute(self, code: str) -> dict[str, Any]:
    """Directly execute code, returning detailed dictionary."""
    logger.debug(f"RestrictedPythonExecutor executing direct call. Code: {code[:100]}...")
    return self._execute(code)
get_code_executor(config)

Creates a code executor instance based on configuration.

Source code in toolboxv2/mods/isaa/base/Agent/executors.py
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
def get_code_executor(config: 'AgentConfig') -> RestrictedPythonExecutor | DockerCodeExecutor | BaseCodeExecutor | None:
    """Creates a code executor instance based on configuration."""
    executor_type = config.code_executor_type
    executor_config = config.code_executor_config or {}

    if executor_type == "restricted":
        if not RESTRICTEDPYTHON_AVAILABLE:
            logger.error("RestrictedPython executor configured but library not installed. Code execution disabled.")
            return None
        return RestrictedPythonExecutor(**executor_config)
    elif executor_type == "docker":
        if not DOCKER_AVAILABLE:
            logger.error("Docker executor configured but library not installed or Docker not running. Code execution disabled.")
            return None
        try:
            return DockerCodeExecutor(**executor_config)
        except Exception as e:
            logger.error(f"Failed to initialize DockerCodeExecutor: {e}. Code execution disabled.")
            return None
    elif executor_type == "none":
        logger.info("Code execution explicitly disabled in configuration.")
        return None
    elif executor_type and ADK_EXEC_AVAILABLE and isinstance(executor_type, BaseCodeExecutor):
        # Allow passing a pre-configured ADK executor instance
        logger.info(f"Using pre-configured ADK code executor: {type(executor_type).__name__}")
        return executor_type
    else:
        logger.warning(f"Unknown or unsupported code_executor_type: '{executor_type}'. Code execution disabled.")
        return None
types
AgentCheckpoint dataclass

Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
@dataclass
class AgentCheckpoint:
    """Enhanced AgentCheckpoint with UnifiedContextManager and ChatSession integration"""
    timestamp: datetime
    agent_state: dict[str, Any]
    task_state: dict[str, Any]
    world_model: dict[str, Any]
    active_flows: list[str]
    metadata: dict[str, Any] = field(default_factory=dict)

    # NEUE: Enhanced checkpoint data for UnifiedContextManager integration
    session_data: dict[str, Any] = field(default_factory=dict)
    context_manager_state: dict[str, Any] = field(default_factory=dict)
    conversation_history: list[dict[str, Any]] = field(default_factory=list)
    variable_system_state: dict[str, Any] = field(default_factory=dict)
    results_store: dict[str, Any] = field(default_factory=dict)
    tool_capabilities: dict[str, Any] = field(default_factory=dict)
    variable_scopes: dict[str, Any] = field(default_factory=dict)

    # Optional: Additional system state
    performance_metrics: dict[str, Any] = field(default_factory=dict)
    execution_history: list[dict[str, Any]] = field(default_factory=list)

    def get_checkpoint_summary(self) -> str:
        """Get human-readable checkpoint summary"""
        try:
            summary_parts = []

            # Basic info
            if self.session_data:
                session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
                summary_parts.append(f"{session_count} sessions")

            # Task info
            if self.task_state:
                completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
                total_tasks = len(self.task_state)
                summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

            # Conversation info
            if self.conversation_history:
                summary_parts.append(f"{len(self.conversation_history)} messages")

            # Context info
            if self.context_manager_state:
                cache_count = self.context_manager_state.get("cache_entries", 0)
                if cache_count > 0:
                    summary_parts.append(f"{cache_count} cached contexts")

            # Variable system info
            if self.variable_system_state:
                scopes = len(self.variable_system_state.get("scopes", {}))
                summary_parts.append(f"{scopes} variable scopes")

            # Tool capabilities
            if self.tool_capabilities:
                summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

            return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

        except Exception as e:
            return f"Summary generation failed: {str(e)}"

    def get_storage_size_estimate(self) -> dict[str, int]:
        """Estimate storage size of different checkpoint components"""
        try:
            sizes = {}

            # Calculate sizes in bytes (approximate)
            sizes["agent_state"] = len(str(self.agent_state))
            sizes["task_state"] = len(str(self.task_state))
            sizes["world_model"] = len(str(self.world_model))
            sizes["conversation_history"] = len(str(self.conversation_history))
            sizes["session_data"] = len(str(self.session_data))
            sizes["context_manager_state"] = len(str(self.context_manager_state))
            sizes["variable_system_state"] = len(str(self.variable_system_state))
            sizes["results_store"] = len(str(self.results_store))
            sizes["tool_capabilities"] = len(str(self.tool_capabilities))

            sizes["total_bytes"] = sum(sizes.values())
            sizes["total_kb"] = sizes["total_bytes"] / 1024
            sizes["total_mb"] = sizes["total_kb"] / 1024

            return sizes

        except Exception as e:
            return {"error": str(e)}

    def validate_checkpoint_integrity(self) -> dict[str, Any]:
        """Validate checkpoint integrity and completeness"""
        validation = {
            "is_valid": True,
            "errors": [],
            "warnings": [],
            "completeness_score": 0.0,
            "components_present": []
        }

        try:
            # Check required components
            required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
            for component in required_components:
                if hasattr(self, component) and getattr(self, component) is not None:
                    validation["components_present"].append(component)
                else:
                    validation["errors"].append(f"Missing required component: {component}")
                    validation["is_valid"] = False

            # Check optional enhanced components
            enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                                   "variable_system_state", "results_store", "tool_capabilities"]

            for component in enhanced_components:
                if hasattr(self, component) and getattr(self, component):
                    validation["components_present"].append(component)

            # Calculate completeness score
            total_possible = len(required_components) + len(enhanced_components)
            validation["completeness_score"] = len(validation["components_present"]) / total_possible

            # Check timestamp validity
            if isinstance(self.timestamp, datetime):
                age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
                if age_hours > 24:
                    validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
            else:
                validation["errors"].append("Invalid timestamp format")
                validation["is_valid"] = False

            # Check session data consistency
            if self.session_data and self.conversation_history:
                session_ids_in_data = set(self.session_data.keys())
                session_ids_in_conversation = set(
                    msg.get("session_id") for msg in self.conversation_history
                    if msg.get("session_id")
                )

                if session_ids_in_data != session_ids_in_conversation:
                    validation["warnings"].append("Session data and conversation history session IDs don't match")

            return validation

        except Exception as e:
            validation["errors"].append(f"Validation error: {str(e)}")
            validation["is_valid"] = False
            return validation

    def get_version_info(self) -> dict[str, str]:
        """Get checkpoint version information"""
        return {
            "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
            "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
            "context_system": "unified" if self.context_manager_state else "legacy",
            "variable_system": "integrated" if self.variable_system_state else "basic",
            "session_management": "chatsession" if self.session_data else "memory_only",
            "created_with": "FlowAgent v2.0 Enhanced Context System"
        }
get_checkpoint_summary()

Get human-readable checkpoint summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
def get_checkpoint_summary(self) -> str:
    """Get human-readable checkpoint summary"""
    try:
        summary_parts = []

        # Basic info
        if self.session_data:
            session_count = len([s for s in self.session_data.values() if s.get("status") != "failed"])
            summary_parts.append(f"{session_count} sessions")

        # Task info
        if self.task_state:
            completed_tasks = len([t for t in self.task_state.values() if t.get("status") == "completed"])
            total_tasks = len(self.task_state)
            summary_parts.append(f"{completed_tasks}/{total_tasks} tasks")

        # Conversation info
        if self.conversation_history:
            summary_parts.append(f"{len(self.conversation_history)} messages")

        # Context info
        if self.context_manager_state:
            cache_count = self.context_manager_state.get("cache_entries", 0)
            if cache_count > 0:
                summary_parts.append(f"{cache_count} cached contexts")

        # Variable system info
        if self.variable_system_state:
            scopes = len(self.variable_system_state.get("scopes", {}))
            summary_parts.append(f"{scopes} variable scopes")

        # Tool capabilities
        if self.tool_capabilities:
            summary_parts.append(f"{len(self.tool_capabilities)} analyzed tools")

        return "; ".join(summary_parts) if summary_parts else "Basic checkpoint"

    except Exception as e:
        return f"Summary generation failed: {str(e)}"
get_storage_size_estimate()

Estimate storage size of different checkpoint components

Source code in toolboxv2/mods/isaa/base/Agent/types.py
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
def get_storage_size_estimate(self) -> dict[str, int]:
    """Estimate storage size of different checkpoint components"""
    try:
        sizes = {}

        # Calculate sizes in bytes (approximate)
        sizes["agent_state"] = len(str(self.agent_state))
        sizes["task_state"] = len(str(self.task_state))
        sizes["world_model"] = len(str(self.world_model))
        sizes["conversation_history"] = len(str(self.conversation_history))
        sizes["session_data"] = len(str(self.session_data))
        sizes["context_manager_state"] = len(str(self.context_manager_state))
        sizes["variable_system_state"] = len(str(self.variable_system_state))
        sizes["results_store"] = len(str(self.results_store))
        sizes["tool_capabilities"] = len(str(self.tool_capabilities))

        sizes["total_bytes"] = sum(sizes.values())
        sizes["total_kb"] = sizes["total_bytes"] / 1024
        sizes["total_mb"] = sizes["total_kb"] / 1024

        return sizes

    except Exception as e:
        return {"error": str(e)}
get_version_info()

Get checkpoint version information

Source code in toolboxv2/mods/isaa/base/Agent/types.py
699
700
701
702
703
704
705
706
707
708
def get_version_info(self) -> dict[str, str]:
    """Get checkpoint version information"""
    return {
        "checkpoint_version": self.metadata.get("checkpoint_version", "1.0"),
        "data_format": "enhanced" if self.session_data or self.context_manager_state else "basic",
        "context_system": "unified" if self.context_manager_state else "legacy",
        "variable_system": "integrated" if self.variable_system_state else "basic",
        "session_management": "chatsession" if self.session_data else "memory_only",
        "created_with": "FlowAgent v2.0 Enhanced Context System"
    }
validate_checkpoint_integrity()

Validate checkpoint integrity and completeness

Source code in toolboxv2/mods/isaa/base/Agent/types.py
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
def validate_checkpoint_integrity(self) -> dict[str, Any]:
    """Validate checkpoint integrity and completeness"""
    validation = {
        "is_valid": True,
        "errors": [],
        "warnings": [],
        "completeness_score": 0.0,
        "components_present": []
    }

    try:
        # Check required components
        required_components = ["timestamp", "agent_state", "task_state", "world_model", "active_flows"]
        for component in required_components:
            if hasattr(self, component) and getattr(self, component) is not None:
                validation["components_present"].append(component)
            else:
                validation["errors"].append(f"Missing required component: {component}")
                validation["is_valid"] = False

        # Check optional enhanced components
        enhanced_components = ["session_data", "context_manager_state", "conversation_history",
                               "variable_system_state", "results_store", "tool_capabilities"]

        for component in enhanced_components:
            if hasattr(self, component) and getattr(self, component):
                validation["components_present"].append(component)

        # Calculate completeness score
        total_possible = len(required_components) + len(enhanced_components)
        validation["completeness_score"] = len(validation["components_present"]) / total_possible

        # Check timestamp validity
        if isinstance(self.timestamp, datetime):
            age_hours = (datetime.now() - self.timestamp).total_seconds() / 3600
            if age_hours > 24:
                validation["warnings"].append(f"Checkpoint is {age_hours:.1f} hours old")
        else:
            validation["errors"].append("Invalid timestamp format")
            validation["is_valid"] = False

        # Check session data consistency
        if self.session_data and self.conversation_history:
            session_ids_in_data = set(self.session_data.keys())
            session_ids_in_conversation = set(
                msg.get("session_id") for msg in self.conversation_history
                if msg.get("session_id")
            )

            if session_ids_in_data != session_ids_in_conversation:
                validation["warnings"].append("Session data and conversation history session IDs don't match")

        return validation

    except Exception as e:
        validation["errors"].append(f"Validation error: {str(e)}")
        validation["is_valid"] = False
        return validation
AgentModelData

Bases: BaseModel

Source code in toolboxv2/mods/isaa/base/Agent/types.py
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
class AgentModelData(BaseModel):
    name: str = "FlowAgent"
    fast_llm_model: str = "openrouter/anthropic/claude-3-haiku"
    complex_llm_model: str = "openrouter/openai/gpt-4o"
    system_message: str = "You are a production-ready autonomous agent."
    temperature: float = 0.7
    max_tokens: int = 2048
    max_input_tokens: int = 32768
    api_key: str | None  = None
    api_base: str | None  = None
    budget_manager: Any  = None
    caching: bool = True
    persona: PersonaConfig | None = True
    use_fast_response: bool = True

    def get_system_message_with_persona(self) -> str:
        """Get system message with persona integration"""
        base_message = self.system_message

        if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
            persona_addition = self.persona.to_system_prompt_addition()
            if persona_addition:
                base_message += f"\n## Persona Instructions\n{persona_addition}"

        return base_message
get_system_message_with_persona()

Get system message with persona integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
783
784
785
786
787
788
789
790
791
792
def get_system_message_with_persona(self) -> str:
    """Get system message with persona integration"""
    base_message = self.system_message

    if self.persona and self.persona.apply_method in ["system_prompt", "both"]:
        persona_addition = self.persona.to_system_prompt_addition()
        if persona_addition:
            base_message += f"\n## Persona Instructions\n{persona_addition}"

    return base_message
ChainMetadata dataclass

Metadata for stored chains

Source code in toolboxv2/mods/isaa/base/Agent/types.py
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
@dataclass
class ChainMetadata:
    """Metadata for stored chains"""
    name: str
    description: str = ""
    created_at: datetime = field(default_factory=datetime.now)
    modified_at: datetime = field(default_factory=datetime.now)
    version: str = "1.0.0"
    tags: list[str] = field(default_factory=list)
    author: str = ""
    complexity: str = "simple"  # simple, medium, complex
    agent_count: int = 0
    has_conditionals: bool = False
    has_parallels: bool = False
    has_error_handling: bool = False
DecisionTask dataclass

Bases: Task

Task für dynamisches Routing

Source code in toolboxv2/mods/isaa/base/Agent/types.py
498
499
500
501
502
503
@dataclass
class DecisionTask(Task):
    """Task für dynamisches Routing"""
    decision_prompt: str = ""  # Kurze Frage an LLM
    routing_map: dict[str, str] = field(default_factory=dict)  # Ergebnis -> nächster Task
    decision_model: str = "fast"  # Welches LLM für Entscheidung
FormatConfig dataclass

Konfiguration für Response-Format und -Länge

Source code in toolboxv2/mods/isaa/base/Agent/types.py
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
@dataclass
class FormatConfig:
    """Konfiguration für Response-Format und -Länge"""
    response_format: ResponseFormat = ResponseFormat.FREE_TEXT
    text_length: TextLength = TextLength.CHAT_CONVERSATION
    custom_instructions: str = ""
    strict_format_adherence: bool = True
    quality_threshold: float = 0.7

    def get_format_instructions(self) -> str:
        """Generiere Format-spezifische Anweisungen"""
        format_instructions = {
            ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
            ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
            ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
            ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
            ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
            ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
            ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
            ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
            ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
            ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
        }
        return format_instructions.get(self.response_format, "Standard-Formatierung.")

    def get_length_instructions(self) -> str:
        """Generiere Längen-spezifische Anweisungen"""
        length_instructions = {
            TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
            TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
            TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
            TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
            TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
        }
        return length_instructions.get(self.text_length, "Standard-Länge.")

    def get_combined_instructions(self) -> str:
        """Kombiniere Format- und Längen-Anweisungen"""
        instructions = []
        instructions.append("## Format-Anforderungen:")
        instructions.append(self.get_format_instructions())
        instructions.append("\n## Längen-Anforderungen:")
        instructions.append(self.get_length_instructions())

        if self.custom_instructions:
            instructions.append("\n## Zusätzliche Anweisungen:")
            instructions.append(self.custom_instructions)

        if self.strict_format_adherence:
            instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

        return "\n".join(instructions)

    def get_expected_word_range(self) -> tuple[int, int]:
        """Erwartete Wortanzahl für Qualitätsbewertung"""
        ranges = {
            TextLength.MINI_CHAT: (10, 50),
            TextLength.CHAT_CONVERSATION: (50, 150),
            TextLength.TABLE_CONVERSATION: (100, 250),
            TextLength.DETAILED_INDEPTH: (300, 800),
            TextLength.PHD_LEVEL: (800, 2000)
        }
        return ranges.get(self.text_length, (50, 200))
get_combined_instructions()

Kombiniere Format- und Längen-Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
def get_combined_instructions(self) -> str:
    """Kombiniere Format- und Längen-Anweisungen"""
    instructions = []
    instructions.append("## Format-Anforderungen:")
    instructions.append(self.get_format_instructions())
    instructions.append("\n## Längen-Anforderungen:")
    instructions.append(self.get_length_instructions())

    if self.custom_instructions:
        instructions.append("\n## Zusätzliche Anweisungen:")
        instructions.append(self.custom_instructions)

    if self.strict_format_adherence:
        instructions.append("\n## ATTENTION: STRICT FORMAT ADHERENCE REQUIRED!")

    return "\n".join(instructions)
get_expected_word_range()

Erwartete Wortanzahl für Qualitätsbewertung

Source code in toolboxv2/mods/isaa/base/Agent/types.py
416
417
418
419
420
421
422
423
424
425
def get_expected_word_range(self) -> tuple[int, int]:
    """Erwartete Wortanzahl für Qualitätsbewertung"""
    ranges = {
        TextLength.MINI_CHAT: (10, 50),
        TextLength.CHAT_CONVERSATION: (50, 150),
        TextLength.TABLE_CONVERSATION: (100, 250),
        TextLength.DETAILED_INDEPTH: (300, 800),
        TextLength.PHD_LEVEL: (800, 2000)
    }
    return ranges.get(self.text_length, (50, 200))
get_format_instructions()

Generiere Format-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
def get_format_instructions(self) -> str:
    """Generiere Format-spezifische Anweisungen"""
    format_instructions = {
        ResponseFormat.FREE_TEXT: "Use natural continuous text without special formatting.",
        ResponseFormat.WITH_TABLES: "Integrate tables for structured data representation. Use Markdown tables.",
        ResponseFormat.WITH_BULLET_POINTS: "Structure information with bullet points (•, -, *) for better readability.",
        ResponseFormat.WITH_LISTS: "Use numbered and unnumbered lists to organize content.",
        ResponseFormat.TEXT_ONLY: "Plain text only without formatting, symbols, or structural elements.",
        ResponseFormat.MD_TEXT: "Full Markdown formatting with headings, code blocks, links, etc.",
        ResponseFormat.YAML_TEXT: "Structure responses in YAML format for machine-readable output.",
        ResponseFormat.JSON_TEXT: "Format responses as a JSON structure for API integration.",
        ResponseFormat.PSEUDO_CODE: "Use pseudocode structure for algorithmic or logical explanations.",
        ResponseFormat.CODE_STRUCTURE: "Structure like code with indentation, comments, and logical blocks."
    }
    return format_instructions.get(self.response_format, "Standard-Formatierung.")
get_length_instructions()

Generiere Längen-spezifische Anweisungen

Source code in toolboxv2/mods/isaa/base/Agent/types.py
388
389
390
391
392
393
394
395
396
397
def get_length_instructions(self) -> str:
    """Generiere Längen-spezifische Anweisungen"""
    length_instructions = {
        TextLength.MINI_CHAT: "Very short, concise answers (1–2 sentences, max 50 words). Chat style.",
        TextLength.CHAT_CONVERSATION: "Moderate conversation length (2–4 sentences, 50–150 words). Natural conversational style.",
        TextLength.TABLE_CONVERSATION: "Structured, tabular presentation with compact explanations (100–250 words).",
        TextLength.DETAILED_INDEPTH: "Comprehensive, detailed explanations (300–800 words) with depth and context.",
        TextLength.PHD_LEVEL: "Academic depth with extensive explanations (800+ words), references, and technical terminology."
    }
    return length_instructions.get(self.text_length, "Standard-Länge.")
LLMTask dataclass

Bases: Task

Spezialisierter Task für LLM-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py
475
476
477
478
479
480
481
482
483
484
485
@dataclass
class LLMTask(Task):
    """Spezialisierter Task für LLM-Aufrufe"""
    llm_config: dict[str, Any] = field(default_factory=lambda: {
        "model_preference": "fast",  # "fast" | "complex"
        "temperature": 0.7,
        "max_tokens": 1024
    })
    prompt_template: str = ""
    context_keys: list[str] = field(default_factory=list)  # Keys aus shared state
    output_schema: dict  = None  # JSON Schema für Validierung
PersonaConfig dataclass
Source code in toolboxv2/mods/isaa/base/Agent/types.py
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
@dataclass
class PersonaConfig:
    name: str
    style: str = "professional"
    personality_traits: list[str] = field(default_factory=lambda: ["helpful", "concise"])
    tone: str = "friendly"
    response_format: str = "direct"
    custom_instructions: str = ""

    format_config: FormatConfig  = None

    apply_method: str = "system_prompt"  # "system_prompt" | "post_process" | "both"
    integration_level: str = "light"  # "light" | "medium" | "heavy"

    def to_system_prompt_addition(self) -> str:
        """Convert persona to system prompt addition with format integration"""
        if self.apply_method in ["system_prompt", "both"]:
            additions = []
            additions.append(f"You are {self.name}.")
            additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

            if self.personality_traits:
                traits_str = ", ".join(self.personality_traits)
                additions.append(f"Your key traits are: {traits_str}.")

            if self.custom_instructions:
                additions.append(self.custom_instructions)

            # Format-spezifische Anweisungen hinzufügen
            if self.format_config:
                additions.append("\n" + self.format_config.get_combined_instructions())

            return " ".join(additions)
        return ""

    def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
        """Dynamische Format-Aktualisierung"""
        try:
            format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
            length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

            if not self.format_config:
                self.format_config = FormatConfig()

            self.format_config.response_format = format_enum
            self.format_config.text_length = length_enum

            if custom_instructions:
                self.format_config.custom_instructions = custom_instructions


        except ValueError:
            raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")

    def should_post_process(self) -> bool:
        """Check if post-processing should be applied"""
        return self.apply_method in ["post_process", "both"]
should_post_process()

Check if post-processing should be applied

Source code in toolboxv2/mods/isaa/base/Agent/types.py
764
765
766
def should_post_process(self) -> bool:
    """Check if post-processing should be applied"""
    return self.apply_method in ["post_process", "both"]
to_system_prompt_addition()

Convert persona to system prompt addition with format integration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
def to_system_prompt_addition(self) -> str:
    """Convert persona to system prompt addition with format integration"""
    if self.apply_method in ["system_prompt", "both"]:
        additions = []
        additions.append(f"You are {self.name}.")
        additions.append(f"Your communication style is {self.style} with a {self.tone} tone.")

        if self.personality_traits:
            traits_str = ", ".join(self.personality_traits)
            additions.append(f"Your key traits are: {traits_str}.")

        if self.custom_instructions:
            additions.append(self.custom_instructions)

        # Format-spezifische Anweisungen hinzufügen
        if self.format_config:
            additions.append("\n" + self.format_config.get_combined_instructions())

        return " ".join(additions)
    return ""
update_format(response_format, text_length, custom_instructions='')

Dynamische Format-Aktualisierung

Source code in toolboxv2/mods/isaa/base/Agent/types.py
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
def update_format(self, response_format: ResponseFormat|str, text_length: TextLength|str, custom_instructions: str = ""):
    """Dynamische Format-Aktualisierung"""
    try:
        format_enum = ResponseFormat(response_format) if isinstance(response_format, str) else response_format
        length_enum = TextLength(text_length) if isinstance(text_length, str) else text_length

        if not self.format_config:
            self.format_config = FormatConfig()

        self.format_config.response_format = format_enum
        self.format_config.text_length = length_enum

        if custom_instructions:
            self.format_config.custom_instructions = custom_instructions


    except ValueError:
        raise ValueError(f"Invalid format '{response_format}' or length '{text_length}'")
PlanData

Bases: BaseModel

Dataclass for plan data

Source code in toolboxv2/mods/isaa/base/Agent/types.py
506
507
508
509
510
511
class PlanData(BaseModel):
    """Dataclass for plan data"""
    plan_name: str = Field(..., discription="Name of the plan")
    description: str = Field(..., discription="Description of the plan")
    execution_strategy: str = Field(..., discription="Execution strategy for the plan")
    tasks: list[LLMTask | ToolTask | DecisionTask] = Field(..., discription="List of tasks in the plan")
ProgressEvent dataclass

Enhanced progress event with better error handling

Source code in toolboxv2/mods/isaa/base/Agent/types.py
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
@dataclass
class ProgressEvent:

    """Enhanced progress event with better error handling"""

    # === 1. Kern-Attribute (Für jedes Event) ===
    event_type: str
    node_name: str
    timestamp: float = field(default_factory=time.time)
    event_id: str = field(default_factory=lambda: str(uuid.uuid4()))
    session_id: Optional[str] = None

    # === 2. Status und Ergebnis-Attribute ===
    status: Optional[NodeStatus] = None
    success: Optional[bool] = None
    duration: Optional[float] = None
    error_details: dict[str, Any] = field(default_factory=dict)  # Strukturiert: message, type, traceback

    # === 3. LLM-spezifische Attribute ===
    llm_model: Optional[str] = None
    llm_prompt_tokens: Optional[int] = None
    llm_completion_tokens: Optional[int] = None
    llm_total_tokens: Optional[int] = None
    llm_cost: Optional[float] = None
    llm_input: Optional[Any] = None  # Optional für Debugging, kann groß sein
    llm_output: Optional[str] = None # Optional für Debugging, kann groß sein

    # === 4. Tool-spezifische Attribute ===
    tool_name: Optional[str] = None
    is_meta_tool: Optional[bool] = None
    tool_args: Optional[dict[str, Any]] = None
    tool_result: Optional[Any] = None
    tool_error: Optional[str] = None
    llm_temperature: Optional[float]  = None

    # === 5. Strategie- und Kontext-Attribute ===
    agent_name: Optional[str] = None
    task_id: Optional[str] = None
    plan_id: Optional[str] = None


    # Node/Routing data
    routing_decision: Optional[str] = None
    node_phase: Optional[str] = None
    node_duration: Optional[float] = None

    # === 6. Metadaten (Für alles andere) ===
    metadata: dict[str, Any] = field(default_factory=dict)


    def __post_init__(self):

        if self.timestamp is None:
            self.timestamp = time.time()

        if self.metadata is None:
            self.metadata = {}
        if not self.event_id:
            self.event_id = f"{self.node_name}_{self.event_type}_{int(self.timestamp * 1000000)}"
        if 'error' in self.metadata or 'error_type' in self.metadata:
            if self.error_details is None:
                self.error_details = {}
            self.error_details['error'] = self.metadata.get('error')
            self.error_details['error_type'] = self.metadata.get('error_type')
            self.status = NodeStatus.FAILED
        if self.status == NodeStatus.FAILED:
            self.success = False
        if self.status == NodeStatus.COMPLETED:
            self.success = True

    def _to_dict(self) -> dict[str, Any]:
        """Convert ProgressEvent to dictionary with proper handling of all field types"""
        result = {}

        # Get all fields from the dataclass
        for field in fields(self):
            value = getattr(self, field.name)

            # Handle None values
            if value is None:
                result[field.name] = None
                continue

            # Handle NodeStatus enum
            if isinstance(value, NodeStatus | Enum):
                result[field.name] = value.value
            # Handle dataclass objects
            elif is_dataclass(value):
                result[field.name] = asdict(value)
            # Handle dictionaries (recursively process nested enums/dataclasses)
            elif isinstance(value, dict):
                result[field.name] = self._process_dict(value)
            # Handle lists (recursively process nested items)
            elif isinstance(value, list):
                result[field.name] = self._process_list(value)
            # Handle primitive types
            else:
                result[field.name] = value

        return result

    def _process_dict(self, d: dict[str, Any]) -> dict[str, Any]:
        """Recursively process dictionary values"""
        result = {}
        for k, v in d.items():
            if isinstance(v, Enum):
                result[k] = v.value
            elif is_dataclass(v):
                result[k] = asdict(v)
            elif isinstance(v, dict):
                result[k] = self._process_dict(v)
            elif isinstance(v, list):
                result[k] = self._process_list(v)
            else:
                result[k] = v
        return result

    def _process_list(self, lst: list[Any]) -> list[Any]:
        """Recursively process list items"""
        result = []
        for item in lst:
            if isinstance(item, Enum):
                result.append(item.value)
            elif is_dataclass(item):
                result.append(asdict(item))
            elif isinstance(item, dict):
                result.append(self._process_dict(item))
            elif isinstance(item, list):
                result.append(self._process_list(item))
            else:
                result.append(item)
        return result

    @classmethod
    def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
        """Create ProgressEvent from dictionary"""
        # Create a copy to avoid modifying the original
        data_copy = dict(data)

        # Handle NodeStatus enum conversion from string back to enum
        if 'status' in data_copy and data_copy['status'] is not None:
            if isinstance(data_copy['status'], str):
                try:
                    data_copy['status'] = NodeStatus(data_copy['status'])
                except (ValueError, TypeError):
                    # If invalid status value, set to None
                    data_copy['status'] = None

        # Filter out any keys that aren't valid dataclass fields
        field_names = {field.name for field in fields(cls)}
        filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

        # Ensure metadata is properly initialized
        if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
            filtered_data['metadata'] = {}

        return cls(**filtered_data)

    def to_dict(self) -> dict[str, Any]:
        """Return event data with None values removed for compact display"""
        data = self._to_dict()

        def clean_dict(d):
            if isinstance(d, dict):
                return {k: clean_dict(v) for k, v in d.items()
                        if v is not None and v != {} and v != [] and v != ''}
            elif isinstance(d, list):
                cleaned_list = [clean_dict(item) for item in d if item is not None]
                return [item for item in cleaned_list if item != {} and item != []]
            return d

        return clean_dict(data)

    def get_chat_display_data(self) -> dict[str, Any]:
        """Get data optimized for chat view display"""
        filtered = self.filter_none_values()

        # Core fields always shown
        core_data = {
            'event_type': filtered.get('event_type'),
            'node_name': filtered.get('node_name'),
            'timestamp': filtered.get('timestamp'),
            'event_id': filtered.get('event_id'),
            'status': filtered.get('status')
        }

        # Add specific fields based on event type
        if self.event_type == 'outline_created':
            if 'metadata' in filtered:
                core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
        elif self.event_type == 'reasoning_loop':
            if 'metadata' in filtered:
                core_data.update({
                    'loop_number': filtered['metadata'].get('loop_number'),
                    'outline_step': filtered['metadata'].get('outline_step'),
                    'context_size': filtered['metadata'].get('context_size')
                })
        elif self.event_type == 'tool_call':
            core_data.update({
                'tool_name': filtered.get('tool_name'),
                'is_meta_tool': filtered.get('is_meta_tool')
            })
        elif self.event_type == 'llm_call':
            core_data.update({
                'llm_model': filtered.get('llm_model'),
                'llm_total_tokens': filtered.get('llm_total_tokens'),
                'llm_cost': filtered.get('llm_cost')
            })

        # Remove None values from core_data
        return {k: v for k, v in core_data.items() if v is not None}

    def get_detailed_display_data(self) -> dict[str, Any]:
        """Get complete filtered data for detailed popup view"""
        return self.filter_none_values()

    def get_progress_summary(self) -> str:
        """Get a brief summary for progress sidebar"""
        if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
            metadata = self.filter_none_values()['metadata']
            loop_num = metadata.get('loop_number', '?')
            step = metadata.get('outline_step', '?')
            return f"Loop {loop_num}, Step {step}"
        elif self.event_type == 'tool_call':
            tool_name = self.tool_name or 'Unknown Tool'
            return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
        elif self.event_type == 'llm_call':
            model = self.llm_model or 'Unknown Model'
            tokens = self.llm_total_tokens
            return f"{model} ({tokens} tokens)" if tokens else model
        else:
            return self.event_type.replace('_', ' ').title()
from_dict(data) classmethod

Create ProgressEvent from dictionary

Source code in toolboxv2/mods/isaa/base/Agent/types.py
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
@classmethod
def from_dict(cls, data: dict[str, Any]) -> 'ProgressEvent':
    """Create ProgressEvent from dictionary"""
    # Create a copy to avoid modifying the original
    data_copy = dict(data)

    # Handle NodeStatus enum conversion from string back to enum
    if 'status' in data_copy and data_copy['status'] is not None:
        if isinstance(data_copy['status'], str):
            try:
                data_copy['status'] = NodeStatus(data_copy['status'])
            except (ValueError, TypeError):
                # If invalid status value, set to None
                data_copy['status'] = None

    # Filter out any keys that aren't valid dataclass fields
    field_names = {field.name for field in fields(cls)}
    filtered_data = {k: v for k, v in data_copy.items() if k in field_names}

    # Ensure metadata is properly initialized
    if 'metadata' not in filtered_data or filtered_data['metadata'] is None:
        filtered_data['metadata'] = {}

    return cls(**filtered_data)
get_chat_display_data()

Get data optimized for chat view display

Source code in toolboxv2/mods/isaa/base/Agent/types.py
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
def get_chat_display_data(self) -> dict[str, Any]:
    """Get data optimized for chat view display"""
    filtered = self.filter_none_values()

    # Core fields always shown
    core_data = {
        'event_type': filtered.get('event_type'),
        'node_name': filtered.get('node_name'),
        'timestamp': filtered.get('timestamp'),
        'event_id': filtered.get('event_id'),
        'status': filtered.get('status')
    }

    # Add specific fields based on event type
    if self.event_type == 'outline_created':
        if 'metadata' in filtered:
            core_data['outline_steps'] = len(filtered['metadata'].get('outline', []))
    elif self.event_type == 'reasoning_loop':
        if 'metadata' in filtered:
            core_data.update({
                'loop_number': filtered['metadata'].get('loop_number'),
                'outline_step': filtered['metadata'].get('outline_step'),
                'context_size': filtered['metadata'].get('context_size')
            })
    elif self.event_type == 'tool_call':
        core_data.update({
            'tool_name': filtered.get('tool_name'),
            'is_meta_tool': filtered.get('is_meta_tool')
        })
    elif self.event_type == 'llm_call':
        core_data.update({
            'llm_model': filtered.get('llm_model'),
            'llm_total_tokens': filtered.get('llm_total_tokens'),
            'llm_cost': filtered.get('llm_cost')
        })

    # Remove None values from core_data
    return {k: v for k, v in core_data.items() if v is not None}
get_detailed_display_data()

Get complete filtered data for detailed popup view

Source code in toolboxv2/mods/isaa/base/Agent/types.py
263
264
265
def get_detailed_display_data(self) -> dict[str, Any]:
    """Get complete filtered data for detailed popup view"""
    return self.filter_none_values()
get_progress_summary()

Get a brief summary for progress sidebar

Source code in toolboxv2/mods/isaa/base/Agent/types.py
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
def get_progress_summary(self) -> str:
    """Get a brief summary for progress sidebar"""
    if self.event_type == 'reasoning_loop' and 'metadata' in self.filter_none_values():
        metadata = self.filter_none_values()['metadata']
        loop_num = metadata.get('loop_number', '?')
        step = metadata.get('outline_step', '?')
        return f"Loop {loop_num}, Step {step}"
    elif self.event_type == 'tool_call':
        tool_name = self.tool_name or 'Unknown Tool'
        return f"{'Meta ' if self.is_meta_tool else ''}{tool_name}"
    elif self.event_type == 'llm_call':
        model = self.llm_model or 'Unknown Model'
        tokens = self.llm_total_tokens
        return f"{model} ({tokens} tokens)" if tokens else model
    else:
        return self.event_type.replace('_', ' ').title()
to_dict()

Return event data with None values removed for compact display

Source code in toolboxv2/mods/isaa/base/Agent/types.py
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def to_dict(self) -> dict[str, Any]:
    """Return event data with None values removed for compact display"""
    data = self._to_dict()

    def clean_dict(d):
        if isinstance(d, dict):
            return {k: clean_dict(v) for k, v in d.items()
                    if v is not None and v != {} and v != [] and v != ''}
        elif isinstance(d, list):
            cleaned_list = [clean_dict(item) for item in d if item is not None]
            return [item for item in cleaned_list if item != {} and item != []]
        return d

    return clean_dict(data)
ProgressTracker

Advanced progress tracking with cost calculation

Source code in toolboxv2/mods/isaa/base/Agent/types.py
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
class ProgressTracker:
    """Advanced progress tracking with cost calculation"""

    def __init__(self, progress_callback: callable  = None, agent_name="unknown"):
        self.progress_callback = progress_callback
        self.events: list[ProgressEvent] = []
        self.active_timers: dict[str, float] = {}

        # Cost tracking (simplified - would need actual provider pricing)
        self.token_costs = {
            "input": 0.00001,  # $0.01/1K tokens input
            "output": 0.00003,  # $0.03/1K tokens output
        }
        self.agent_name = agent_name

    async def emit_event(self, event: ProgressEvent):
        """Emit progress event with callback and storage"""
        self.events.append(event)
        event.agent_name = self.agent_name

        if self.progress_callback:
            try:
                if asyncio.iscoroutinefunction(self.progress_callback):
                    await self.progress_callback(event)
                else:
                    self.progress_callback(event)
            except Exception:
                import traceback
                print(traceback.format_exc())


    def start_timer(self, key: str) -> float:
        """Start timing operation"""
        start_time = time.perf_counter()
        self.active_timers[key] = start_time
        return start_time

    def end_timer(self, key: str) -> float:
        """End timing operation and return duration"""
        if key not in self.active_timers:
            return 0.0
        duration = time.perf_counter() - self.active_timers[key]
        del self.active_timers[key]
        return duration

    def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
        """Calculate approximate LLM cost"""
        cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
        if hasattr(completion_response, "_hidden_params"):
            cost = completion_response._hidden_params.get("response_cost", 0)
        try:
            import litellm
            cost = litellm.completion_cost(model=model, completion_response=completion_response)
        except ImportError:
            pass
        except Exception as e:
            try:
                import litellm
                cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
            except Exception:
                pass
        return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]

    def get_summary(self) -> dict[str, Any]:
        """Get comprehensive progress summary"""
        summary = {
            "total_events": len(self.events),
            "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
            "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
            "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
            "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
            "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
            "nodes_visited": list(set(e.node_name for e in self.events)),
            "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
            "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
        }
        return summary
calculate_llm_cost(model, input_tokens, output_tokens, completion_response=None)

Calculate approximate LLM cost

Source code in toolboxv2/mods/isaa/base/Agent/types.py
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
def calculate_llm_cost(self, model: str, input_tokens: int, output_tokens: int,completion_response:Any=None) -> float:
    """Calculate approximate LLM cost"""
    cost = (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
    if hasattr(completion_response, "_hidden_params"):
        cost = completion_response._hidden_params.get("response_cost", 0)
    try:
        import litellm
        cost = litellm.completion_cost(model=model, completion_response=completion_response)
    except ImportError:
        pass
    except Exception as e:
        try:
            import litellm
            cost = litellm.completion_cost(model=model.split('/')[-1], completion_response=completion_response)
        except Exception:
            pass
    return cost or (input_tokens / 1000) * self.token_costs["input"] + (output_tokens / 1000) * self.token_costs["output"]
emit_event(event) async

Emit progress event with callback and storage

Source code in toolboxv2/mods/isaa/base/Agent/types.py
299
300
301
302
303
304
305
306
307
308
309
310
311
312
async def emit_event(self, event: ProgressEvent):
    """Emit progress event with callback and storage"""
    self.events.append(event)
    event.agent_name = self.agent_name

    if self.progress_callback:
        try:
            if asyncio.iscoroutinefunction(self.progress_callback):
                await self.progress_callback(event)
            else:
                self.progress_callback(event)
        except Exception:
            import traceback
            print(traceback.format_exc())
end_timer(key)

End timing operation and return duration

Source code in toolboxv2/mods/isaa/base/Agent/types.py
321
322
323
324
325
326
327
def end_timer(self, key: str) -> float:
    """End timing operation and return duration"""
    if key not in self.active_timers:
        return 0.0
    duration = time.perf_counter() - self.active_timers[key]
    del self.active_timers[key]
    return duration
get_summary()

Get comprehensive progress summary

Source code in toolboxv2/mods/isaa/base/Agent/types.py
347
348
349
350
351
352
353
354
355
356
357
358
359
360
def get_summary(self) -> dict[str, Any]:
    """Get comprehensive progress summary"""
    summary = {
        "total_events": len(self.events),
        "llm_calls": len([e for e in self.events if e.event_type == "llm_call"]),
        "tool_calls": len([e for e in self.events if e.event_type == "tool_call"]),
        "total_cost": sum(e.llm_cost for e in self.events if e.llm_cost),
        "total_tokens": sum(e.llm_total_tokens for e in self.events if e.llm_total_tokens),
        "total_duration": sum(e.node_duration for e in self.events if e.node_duration),
        "nodes_visited": list(set(e.node_name for e in self.events)),
        "tools_used": list(set(e.tool_name for e in self.events if e.tool_name)),
        "models_used": list(set(e.llm_model for e in self.events if e.llm_model))
    }
    return summary
start_timer(key)

Start timing operation

Source code in toolboxv2/mods/isaa/base/Agent/types.py
315
316
317
318
319
def start_timer(self, key: str) -> float:
    """Start timing operation"""
    start_time = time.perf_counter()
    self.active_timers[key] = start_time
    return start_time
Task dataclass
Source code in toolboxv2/mods/isaa/base/Agent/types.py
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
@dataclass
class Task:
    id: str
    type: str
    description: str
    status: str = "pending"  # pending, running, completed, failed, paused
    priority: int = 1
    dependencies: list[str] = field(default_factory=list)
    subtasks: list[str] = field(default_factory=list)
    result: Any = None
    error: str = None
    created_at: datetime = field(default_factory=datetime.now)
    started_at: datetime  = None
    completed_at: datetime  = None
    metadata: dict[str, Any] = field(default_factory=dict)
    retry_count: int = 0
    max_retries: int = 3
    critical: bool = False

    task_identification_attr: bool = True


    def __post_init__(self):
        """Ensure all mutable defaults are properly initialized"""
        if self.metadata is None:
            self.metadata = {}
        if self.dependencies is None:
            self.dependencies = []
        if self.subtasks is None:
            self.subtasks = []

    def __getitem__(self, key):
        return getattr(self, key)

    def __setitem__(self, key, value):
        setattr(self, key, value)
__post_init__()

Ensure all mutable defaults are properly initialized

Source code in toolboxv2/mods/isaa/base/Agent/types.py
449
450
451
452
453
454
455
456
def __post_init__(self):
    """Ensure all mutable defaults are properly initialized"""
    if self.metadata is None:
        self.metadata = {}
    if self.dependencies is None:
        self.dependencies = []
    if self.subtasks is None:
        self.subtasks = []
ToolAnalysis

Bases: BaseModel

Defines the structure for a valid tool analysis.

Source code in toolboxv2/mods/isaa/base/Agent/types.py
795
796
797
798
799
800
801
802
803
804
805
class ToolAnalysis(BaseModel):
    """Defines the structure for a valid tool analysis."""
    primary_function: str = Field(..., description="The main purpose of the tool.")
    use_cases: list[str] = Field(..., description="Specific use cases for the tool.")
    trigger_phrases: list[str] = Field(..., description="Phrases that should trigger the tool.")
    indirect_connections: list[str] = Field(..., description="Non-obvious connections or applications.")
    complexity_scenarios: list[str] = Field(..., description="Complex scenarios where the tool can be applied.")
    user_intent_categories: list[str] = Field(..., description="Categories of user intent the tool addresses.")
    confidence_triggers: dict[str, float] = Field(..., description="Phrases mapped to confidence scores.")
    tool_complexity: str = Field(..., description="The complexity of the tool, rated as low, medium, or high.")
    args_schema: dict[str, Any] | None = Field(..., description="The schema for the tool's arguments.")
ToolTask dataclass

Bases: Task

Spezialisierter Task für Tool-Aufrufe

Source code in toolboxv2/mods/isaa/base/Agent/types.py
488
489
490
491
492
493
494
495
@dataclass
class ToolTask(Task):
    """Spezialisierter Task für Tool-Aufrufe"""
    tool_name: str = ""
    arguments: dict[str, Any] = field(default_factory=dict)  # Kann {{ }} Referenzen enthalten
    hypothesis: str = ""  # Was erwarten wir von diesem Tool?
    validation_criteria: str = ""  # Wie validieren wir das Ergebnis?
    expectation: str = ""  # Wie sollte das Ergebnis aussehen?
create_task(task_type, **kwargs)

Factory für Task-Erstellung mit korrektem Typ

Source code in toolboxv2/mods/isaa/base/Agent/types.py
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
def create_task(task_type: str, **kwargs) -> Task:
    """Factory für Task-Erstellung mit korrektem Typ"""
    task_classes = {
        "llm_call": LLMTask,
        "tool_call": ToolTask,
        "decision": DecisionTask,
        "generic": Task,
        "LLMTask": LLMTask,
        "ToolTask": ToolTask,
        "DecisionTask": DecisionTask,
        "Task": Task,
    }

    task_class = task_classes.get(task_type, Task)

    # Standard-Felder setzen
    if "id" not in kwargs:
        kwargs["id"] = str(uuid.uuid4())
    if "type" not in kwargs:
        kwargs["type"] = task_type
    if "critical" not in kwargs:
        kwargs["critical"] = task_type in ["llm_call", "decision"]

    # Ensure metadata is initialized
    if "metadata" not in kwargs:
        kwargs["metadata"] = {}

    # Create task and ensure post_init is called
    task = task_class(**kwargs)

    # Double-check metadata initialization
    if not hasattr(task, 'metadata') or task.metadata is None:
        task.metadata = {}

    return task
utils
LLMMessage dataclass

Represents a message in a conversation with the LLM.

Source code in toolboxv2/mods/isaa/base/Agent/utils.py
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
@dataclass
class LLMMessage:
    """Represents a message in a conversation with the LLM."""
    role: str  # "user", "assistant", "system", "tool"
    # Content can be string or list (e.g., multimodal with text/image dicts)
    # Conforms to LiteLLM/OpenAI structure
    content: str | list[dict[str, Any]]
    tool_call_id: str | None = None  # For tool responses
    name: str | None = None  # For tool calls/responses (function name)

    def to_dict(self) -> dict:
        """Convert to dictionary, handling potential dataclass nuances."""
        d = {"role": self.role, "content": self.content}
        if self.tool_call_id:
            d["tool_call_id"] = self.tool_call_id
        if self.name:
            d["name"] = self.name
        return d
to_dict()

Convert to dictionary, handling potential dataclass nuances.

Source code in toolboxv2/mods/isaa/base/Agent/utils.py
144
145
146
147
148
149
150
151
def to_dict(self) -> dict:
    """Convert to dictionary, handling potential dataclass nuances."""
    d = {"role": self.role, "content": self.content}
    if self.tool_call_id:
        d["tool_call_id"] = self.tool_call_id
    if self.name:
        d["name"] = self.name
    return d
WorldModel dataclass

Thread-safe representation of the agent's persistent understanding of the world.

Source code in toolboxv2/mods/isaa/base/Agent/utils.py
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
@dataclass
class WorldModel:
    """Thread-safe representation of the agent's persistent understanding of the world."""
    data: dict[str, Any] = dataclass_field(default_factory=dict)
    _lock: threading.Lock = dataclass_field(default_factory=threading.Lock)

    def get(self, key: str, default: Any = None) -> Any:
        with self._lock:
            return self.data.get(key, default)

    def set(self, key: str, value: Any):
        with self._lock:
            logger_wm.debug(f"WorldModel SET: {key} = {value}")
            self.data[key] = value

    def remove(self, key: str):
        with self._lock:
            if key in self.data:
                logger_wm.debug(f"WorldModel REMOVE: {key}")
                del self.data[key]

    def show(self) -> str:
        with self._lock:
            if not self.data:
                return "[empty]"
            try:
                items = [f"- {k}: {json.dumps(v, indent=None, ensure_ascii=False, default=str)}"
                         for k, v in self.data.items()]
                return "\n".join(items)
            except Exception:
                items = [f"- {k}: {str(v)}" for k, v in self.data.items()]
                return "\n".join(items)

    def to_dict(self) -> dict[str, Any]:
        with self._lock:
            # Deep copy might be needed if values are mutable and modified externally
            # For simplicity, shallow copy is used here.
            return self.data.copy()

    def update_from_dict(self, data_dict: dict[str, Any]):
        with self._lock:
            self.data.update(data_dict)
            logger_wm.debug(f"WorldModel updated from dict: {list(data_dict.keys())}")
AgentUtils
AISemanticMemory
Source code in toolboxv2/mods/isaa/base/AgentUtils.py
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
class AISemanticMemory(metaclass=Singleton):
    def __init__(self,
                 base_path: str = "/semantic_memory",
                 default_model: str = os.getenv("BLITZMODEL"),
                 default_embedding_model: str = os.getenv("DEFAULTMODELEMBEDDING"),
                 default_similarity_threshold: float = 0.61,
                 default_batch_size: int = 64,
                 default_n_clusters: int = 2,
                 default_deduplication_threshold: float = 0.85):
        """
        Initialize AISemanticMemory with KnowledgeBase integration

        Args:
            base_path: Root directory for memory storage
            default_model: Default model for text generation
            default_embedding_model: Default embedding model
            default_similarity_threshold: Default similarity threshold for retrieval
            default_batch_size: Default batch size for processing
            default_n_clusters: Default number of clusters for FAISS
            default_deduplication_threshold: Default threshold for deduplication
        """
        self.base_path = os.path.join(os.getcwd(), ".data", base_path)
        self.memories: dict[str, KnowledgeBase] = {}

        # Map of embedding models to their dimensions
        self.embedding_dims = {
            "text-embedding-3-small": 1536,
            "text-embedding-3-large": 3072,
            "nomic-embed-text": 768,
            "default": 768
        }

        self.default_config = {
            "embedding_model": default_embedding_model,
            "embedding_dim": self._get_embedding_dim(default_embedding_model),
            "similarity_threshold": default_similarity_threshold,
            "batch_size": default_batch_size,
            "n_clusters": default_n_clusters,
            "deduplication_threshold": default_deduplication_threshold,
            "model_name": default_model
        }

    def _get_embedding_dim(self, model_name: str) -> int:
        """Get embedding dimension for a model"""
        return self.embedding_dims.get(model_name, 768)

    @staticmethod
    def _sanitize_name(name: str) -> str:
        """Sanitize memory name for filesystem safety"""
        name = re.sub(r'[^a-zA-Z0-9_-]', '-', name)[:63].strip('-')
        if not name:
            raise ValueError("Invalid memory name")
        if len(name) < 3:
            name += "Z" * (3 - len(name))
        return name

    def create_memory(self,
                      name: str,
                      model_config: dict | None = None,
                      storage_config: dict | None = None) -> KnowledgeBase:
        """
        Create new memory store with KnowledgeBase

        Args:
            name: Unique name for the memory store
            model_config: Configuration for embedding model
            storage_config: Configuration for KnowledgeBase parameters
        """
        sanitized = self._sanitize_name(name)
        if sanitized in self.memories:
            raise ValueError(f"Memory '{name}' already exists")

        # Determine embedding model and dimension
        embedding_model = self.default_config["embedding_model"]
        model_name = self.default_config["model_name"]
        if model_config:
            embedding_model = model_config.get("embedding_model", embedding_model)
            model_name = model_config.get("model_name", model_name)
        embedding_dim = self._get_embedding_dim(embedding_model)

        # Get KnowledgeBase parameters
        kb_params = {
            "embedding_dim": embedding_dim,
            "embedding_model": embedding_model,
            "similarity_threshold": self.default_config["similarity_threshold"],
            "batch_size": self.default_config["batch_size"],
            "n_clusters": self.default_config["n_clusters"],
            "deduplication_threshold": self.default_config["deduplication_threshold"],
            "model_name": model_name,
        }

        if storage_config:
            kb_params.update({
                "similarity_threshold": storage_config.get("similarity_threshold", kb_params["similarity_threshold"]),
                "batch_size": storage_config.get("batch_size", kb_params["batch_size"]),
                "n_clusters": storage_config.get("n_clusters", kb_params["n_clusters"]),
                "model_name": storage_config.get("model_name", kb_params["model_name"]),
                "embedding_model": storage_config.get("embedding_model", kb_params["embedding_model"]),
                "deduplication_threshold": storage_config.get("deduplication_threshold",
                                                              kb_params["deduplication_threshold"]),
            })

        # Create KnowledgeBase instance
        self.memories[sanitized] = KnowledgeBase(**kb_params)
        return self.memories[sanitized]

    async def add_data(self,
                       memory_name: str,
                       data: str | list[str] | bytes | dict,
                       metadata: dict | None = None, direct=False) -> bool:
        """
        Add data to memory store

        Args:
            memory_name: Target memory store
            data: Text, list of texts, binary file, or structured data
            metadata: Optional metadata
        """
        name = self._sanitize_name(memory_name)
        kb = self.memories.get(name)
        if not kb:
            kb = self.create_memory(name)

        # Process input data
        texts = []
        if isinstance(data, bytes):
            try:
                text = extract_text_natively(data, filename="" if metadata is None else metadata.get("filename", ""))
                texts = [text.replace('\\t', '').replace('\t', '')]
            except Exception as e:
                raise ValueError(f"File processing failed: {str(e)}")
        elif isinstance(data, str):
            texts = [data.replace('\\t', '').replace('\t', '')]
        elif isinstance(data, list):
            texts = [d.replace('\\t', '').replace('\t', '') for d in data]
        elif isinstance(data, dict):
            # Custom KG not supported in current KnowledgeBase
            raise NotImplementedError("Custom knowledge graph insertion not supported")
        else:
            raise ValueError("Unsupported data type")

        # Add data to KnowledgeBase
        try:
            added, duplicates = await kb.add_data(texts, metadata, direct=direct)
            return added > 0
        except Exception as e:
            import traceback
            print(traceback.format_exc())
            raise RuntimeError(f"Data addition failed: {str(e)}")

    def get(self, names):
        return [m for n,m in self._get_target_memories(names)]

    async def query(self,
                    query: str,
                    memory_names: str | list[str] | None = None,
                    query_params: dict | None = None,
                    to_str: bool = False,
                    unified_retrieve: bool =False) -> str | list[dict]:
        """
        Query memories using KnowledgeBase retrieval

        Args:
            query: Search query
            memory_names: Target memory names
            query_params: Query parameters
            to_str: Return string format
            unified_retrieve: Unified retrieve
        """
        targets = self._get_target_memories(memory_names)
        if not targets:
            return []

        results = []
        for name, kb in targets:
            #try:
                # Use KnowledgeBase's retrieve_with_overview for comprehensive results
                result = await kb.retrieve_with_overview(
                    query=query,
                    k=query_params.get("k", 3) if query_params else 3,
                    min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                    cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                    max_cross_refs=query_params.get("max_cross_refs", 2) if query_params else 2,
                    max_sentences=query_params.get("max_sentences", 5) if query_params else 5
                ) if not unified_retrieve else await kb.unified_retrieve(
                    query=query,
                    k=query_params.get("k", 2) if query_params else 2,
                    min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                    cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                    max_cross_refs=query_params.get("max_cross_refs", 6) if query_params else 6,
                    max_sentences=query_params.get("max_sentences", 12) if query_params else 12
                )
                if result.overview:
                    results.append({
                        "memory": name,
                        "result": result
                    })
            #except Exception as e:
            #    print(f"Query failed on {name}: {str(e)}")
        if to_str:
            str_res = ""
            if not unified_retrieve:
                str_res = [
                    f"{x['memory']} - {json.dumps(x['result'].overview)}\n - {[c.text for c in x['result'].details]}\n - {[(k, [c.text for c in v]) for k, v in x['result'].cross_references.items()]}"
                    for x in results]
                # str_res =
            else:
                str_res = json.dumps(results)
            return str_res
        return results

    def _get_target_memories(self, memory_names: str | list[str] | None) -> list[tuple[str, KnowledgeBase]]:
        """Get target memories for query"""
        if not memory_names:
            return list(self.memories.items())

        names = [memory_names] if isinstance(memory_names, str) else memory_names

        targets = []
        for name in names:
            sanitized = self._sanitize_name(name)
            if kb := self.memories.get(sanitized):
                targets.append((sanitized, kb))
        return targets

    def list_memories(self) -> list[str]:
        """List all available memories"""
        return list(self.memories.keys())

    async def delete_memory(self, name: str) -> bool:
        """Delete a memory store"""
        sanitized = self._sanitize_name(name)
        if sanitized in self.memories:
            del self.memories[sanitized]
            return True
        return False

    def save_memory(self, name: str, path: str) -> bool | bytes:
        """Save a memory store to disk"""
        sanitized = self._sanitize_name(name)
        if kb := self.memories.get(sanitized):
            try:
                return kb.save(path)
            except Exception as e:
                print(f"Error saving memory: {str(e)}")
                return False
        return False

    def save_all_memories(self, path: str) -> bool:
        """Save all memory stores to disk"""
        for name, kb in self.memories.items():
            try:
                kb.save(os.path.join(path, f"{name}.pkl"))
            except Exception as e:
                print(f"Error saving memory: {str(e)}")
                return False
        return True

    def load_all_memories(self, path: str) -> bool:
        """Load all memory stores from disk"""
        for file in os.listdir(path):
            if file.endswith(".pkl"):
                try:
                    self.memories[file[:-4]] = KnowledgeBase.load(os.path.join(path, file))
                except EOFError:
                    return False
                except FileNotFoundError:
                    return False
                except Exception as e:
                    print(f"Error loading memory: {str(e)}")
                    return False
        return True

    def load_memory(self, name: str, path: str | bytes) -> bool:
        """Load a memory store from disk"""
        sanitized = self._sanitize_name(name)
        if sanitized in self.memories:
            return False
        try:
            self.memories[sanitized] = KnowledgeBase.load(path)
            return True
        except Exception:
            # print(f"Error loading memory: {str(e)}")
            return False
__init__(base_path='/semantic_memory', default_model=os.getenv('BLITZMODEL'), default_embedding_model=os.getenv('DEFAULTMODELEMBEDDING'), default_similarity_threshold=0.61, default_batch_size=64, default_n_clusters=2, default_deduplication_threshold=0.85)

Initialize AISemanticMemory with KnowledgeBase integration

Parameters:

Name Type Description Default
base_path str

Root directory for memory storage

'/semantic_memory'
default_model str

Default model for text generation

getenv('BLITZMODEL')
default_embedding_model str

Default embedding model

getenv('DEFAULTMODELEMBEDDING')
default_similarity_threshold float

Default similarity threshold for retrieval

0.61
default_batch_size int

Default batch size for processing

64
default_n_clusters int

Default number of clusters for FAISS

2
default_deduplication_threshold float

Default threshold for deduplication

0.85
Source code in toolboxv2/mods/isaa/base/AgentUtils.py
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
def __init__(self,
             base_path: str = "/semantic_memory",
             default_model: str = os.getenv("BLITZMODEL"),
             default_embedding_model: str = os.getenv("DEFAULTMODELEMBEDDING"),
             default_similarity_threshold: float = 0.61,
             default_batch_size: int = 64,
             default_n_clusters: int = 2,
             default_deduplication_threshold: float = 0.85):
    """
    Initialize AISemanticMemory with KnowledgeBase integration

    Args:
        base_path: Root directory for memory storage
        default_model: Default model for text generation
        default_embedding_model: Default embedding model
        default_similarity_threshold: Default similarity threshold for retrieval
        default_batch_size: Default batch size for processing
        default_n_clusters: Default number of clusters for FAISS
        default_deduplication_threshold: Default threshold for deduplication
    """
    self.base_path = os.path.join(os.getcwd(), ".data", base_path)
    self.memories: dict[str, KnowledgeBase] = {}

    # Map of embedding models to their dimensions
    self.embedding_dims = {
        "text-embedding-3-small": 1536,
        "text-embedding-3-large": 3072,
        "nomic-embed-text": 768,
        "default": 768
    }

    self.default_config = {
        "embedding_model": default_embedding_model,
        "embedding_dim": self._get_embedding_dim(default_embedding_model),
        "similarity_threshold": default_similarity_threshold,
        "batch_size": default_batch_size,
        "n_clusters": default_n_clusters,
        "deduplication_threshold": default_deduplication_threshold,
        "model_name": default_model
    }
add_data(memory_name, data, metadata=None, direct=False) async

Add data to memory store

Parameters:

Name Type Description Default
memory_name str

Target memory store

required
data str | list[str] | bytes | dict

Text, list of texts, binary file, or structured data

required
metadata dict | None

Optional metadata

None
Source code in toolboxv2/mods/isaa/base/AgentUtils.py
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
async def add_data(self,
                   memory_name: str,
                   data: str | list[str] | bytes | dict,
                   metadata: dict | None = None, direct=False) -> bool:
    """
    Add data to memory store

    Args:
        memory_name: Target memory store
        data: Text, list of texts, binary file, or structured data
        metadata: Optional metadata
    """
    name = self._sanitize_name(memory_name)
    kb = self.memories.get(name)
    if not kb:
        kb = self.create_memory(name)

    # Process input data
    texts = []
    if isinstance(data, bytes):
        try:
            text = extract_text_natively(data, filename="" if metadata is None else metadata.get("filename", ""))
            texts = [text.replace('\\t', '').replace('\t', '')]
        except Exception as e:
            raise ValueError(f"File processing failed: {str(e)}")
    elif isinstance(data, str):
        texts = [data.replace('\\t', '').replace('\t', '')]
    elif isinstance(data, list):
        texts = [d.replace('\\t', '').replace('\t', '') for d in data]
    elif isinstance(data, dict):
        # Custom KG not supported in current KnowledgeBase
        raise NotImplementedError("Custom knowledge graph insertion not supported")
    else:
        raise ValueError("Unsupported data type")

    # Add data to KnowledgeBase
    try:
        added, duplicates = await kb.add_data(texts, metadata, direct=direct)
        return added > 0
    except Exception as e:
        import traceback
        print(traceback.format_exc())
        raise RuntimeError(f"Data addition failed: {str(e)}")
create_memory(name, model_config=None, storage_config=None)

Create new memory store with KnowledgeBase

Parameters:

Name Type Description Default
name str

Unique name for the memory store

required
model_config dict | None

Configuration for embedding model

None
storage_config dict | None

Configuration for KnowledgeBase parameters

None
Source code in toolboxv2/mods/isaa/base/AgentUtils.py
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
def create_memory(self,
                  name: str,
                  model_config: dict | None = None,
                  storage_config: dict | None = None) -> KnowledgeBase:
    """
    Create new memory store with KnowledgeBase

    Args:
        name: Unique name for the memory store
        model_config: Configuration for embedding model
        storage_config: Configuration for KnowledgeBase parameters
    """
    sanitized = self._sanitize_name(name)
    if sanitized in self.memories:
        raise ValueError(f"Memory '{name}' already exists")

    # Determine embedding model and dimension
    embedding_model = self.default_config["embedding_model"]
    model_name = self.default_config["model_name"]
    if model_config:
        embedding_model = model_config.get("embedding_model", embedding_model)
        model_name = model_config.get("model_name", model_name)
    embedding_dim = self._get_embedding_dim(embedding_model)

    # Get KnowledgeBase parameters
    kb_params = {
        "embedding_dim": embedding_dim,
        "embedding_model": embedding_model,
        "similarity_threshold": self.default_config["similarity_threshold"],
        "batch_size": self.default_config["batch_size"],
        "n_clusters": self.default_config["n_clusters"],
        "deduplication_threshold": self.default_config["deduplication_threshold"],
        "model_name": model_name,
    }

    if storage_config:
        kb_params.update({
            "similarity_threshold": storage_config.get("similarity_threshold", kb_params["similarity_threshold"]),
            "batch_size": storage_config.get("batch_size", kb_params["batch_size"]),
            "n_clusters": storage_config.get("n_clusters", kb_params["n_clusters"]),
            "model_name": storage_config.get("model_name", kb_params["model_name"]),
            "embedding_model": storage_config.get("embedding_model", kb_params["embedding_model"]),
            "deduplication_threshold": storage_config.get("deduplication_threshold",
                                                          kb_params["deduplication_threshold"]),
        })

    # Create KnowledgeBase instance
    self.memories[sanitized] = KnowledgeBase(**kb_params)
    return self.memories[sanitized]
delete_memory(name) async

Delete a memory store

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
541
542
543
544
545
546
547
async def delete_memory(self, name: str) -> bool:
    """Delete a memory store"""
    sanitized = self._sanitize_name(name)
    if sanitized in self.memories:
        del self.memories[sanitized]
        return True
    return False
list_memories()

List all available memories

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
537
538
539
def list_memories(self) -> list[str]:
    """List all available memories"""
    return list(self.memories.keys())
load_all_memories(path)

Load all memory stores from disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
570
571
572
573
574
575
576
577
578
579
580
581
582
583
def load_all_memories(self, path: str) -> bool:
    """Load all memory stores from disk"""
    for file in os.listdir(path):
        if file.endswith(".pkl"):
            try:
                self.memories[file[:-4]] = KnowledgeBase.load(os.path.join(path, file))
            except EOFError:
                return False
            except FileNotFoundError:
                return False
            except Exception as e:
                print(f"Error loading memory: {str(e)}")
                return False
    return True
load_memory(name, path)

Load a memory store from disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
585
586
587
588
589
590
591
592
593
594
595
def load_memory(self, name: str, path: str | bytes) -> bool:
    """Load a memory store from disk"""
    sanitized = self._sanitize_name(name)
    if sanitized in self.memories:
        return False
    try:
        self.memories[sanitized] = KnowledgeBase.load(path)
        return True
    except Exception:
        # print(f"Error loading memory: {str(e)}")
        return False
query(query, memory_names=None, query_params=None, to_str=False, unified_retrieve=False) async

Query memories using KnowledgeBase retrieval

Parameters:

Name Type Description Default
query str

Search query

required
memory_names str | list[str] | None

Target memory names

None
query_params dict | None

Query parameters

None
to_str bool

Return string format

False
unified_retrieve bool

Unified retrieve

False
Source code in toolboxv2/mods/isaa/base/AgentUtils.py
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
async def query(self,
                query: str,
                memory_names: str | list[str] | None = None,
                query_params: dict | None = None,
                to_str: bool = False,
                unified_retrieve: bool =False) -> str | list[dict]:
    """
    Query memories using KnowledgeBase retrieval

    Args:
        query: Search query
        memory_names: Target memory names
        query_params: Query parameters
        to_str: Return string format
        unified_retrieve: Unified retrieve
    """
    targets = self._get_target_memories(memory_names)
    if not targets:
        return []

    results = []
    for name, kb in targets:
        #try:
            # Use KnowledgeBase's retrieve_with_overview for comprehensive results
            result = await kb.retrieve_with_overview(
                query=query,
                k=query_params.get("k", 3) if query_params else 3,
                min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                max_cross_refs=query_params.get("max_cross_refs", 2) if query_params else 2,
                max_sentences=query_params.get("max_sentences", 5) if query_params else 5
            ) if not unified_retrieve else await kb.unified_retrieve(
                query=query,
                k=query_params.get("k", 2) if query_params else 2,
                min_similarity=query_params.get("min_similarity", 0.2) if query_params else 0.2,
                cross_ref_depth=query_params.get("cross_ref_depth", 2) if query_params else 2,
                max_cross_refs=query_params.get("max_cross_refs", 6) if query_params else 6,
                max_sentences=query_params.get("max_sentences", 12) if query_params else 12
            )
            if result.overview:
                results.append({
                    "memory": name,
                    "result": result
                })
        #except Exception as e:
        #    print(f"Query failed on {name}: {str(e)}")
    if to_str:
        str_res = ""
        if not unified_retrieve:
            str_res = [
                f"{x['memory']} - {json.dumps(x['result'].overview)}\n - {[c.text for c in x['result'].details]}\n - {[(k, [c.text for c in v]) for k, v in x['result'].cross_references.items()]}"
                for x in results]
            # str_res =
        else:
            str_res = json.dumps(results)
        return str_res
    return results
save_all_memories(path)

Save all memory stores to disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
560
561
562
563
564
565
566
567
568
def save_all_memories(self, path: str) -> bool:
    """Save all memory stores to disk"""
    for name, kb in self.memories.items():
        try:
            kb.save(os.path.join(path, f"{name}.pkl"))
        except Exception as e:
            print(f"Error saving memory: {str(e)}")
            return False
    return True
save_memory(name, path)

Save a memory store to disk

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
549
550
551
552
553
554
555
556
557
558
def save_memory(self, name: str, path: str) -> bool | bytes:
    """Save a memory store to disk"""
    sanitized = self._sanitize_name(name)
    if kb := self.memories.get(sanitized):
        try:
            return kb.save(path)
        except Exception as e:
            print(f"Error saving memory: {str(e)}")
            return False
    return False
PyEnvEval
Source code in toolboxv2/mods/isaa/base/AgentUtils.py
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
class PyEnvEval:
    def __init__(self):
        self.local_env = locals().copy()
        self.global_env = {'local_env': self.local_env}  # globals().copy()

    def eval_code(self, code):
        try:
            exec(code, self.global_env, self.local_env)
            result = eval(code, self.global_env, self.local_env)
            return self.format_output(result)
        except Exception as e:
            return self.format_output(str(e))

    def get_env(self):
        local_env_str = self.format_env(self.local_env)
        return f'Locals:\n{local_env_str}'

    @staticmethod
    def format_output(output):
        return f'Ergebnis: {output}'

    @staticmethod
    def format_env(env):
        return '\n'.join(f'{key}: {value}' for key, value in env.items())

    def run_and_display(self, python_code):
        """function to eval python code"""
        start = f'Start-state:\n{self.get_env()}'
        result = self.eval_code(python_code)
        end = f'End-state:\n{self.get_env()}'
        return f'{start}\nResult:\n{result}\n{end}'

    def tool(self):
        return {"PythonEval": {"func": self.run_and_display, "description": "Use Python Code to Get to an Persis Answer! input must be valid python code all non code parts must be comments!"}}
run_and_display(python_code)

function to eval python code

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
796
797
798
799
800
801
def run_and_display(self, python_code):
    """function to eval python code"""
    start = f'Start-state:\n{self.get_env()}'
    result = self.eval_code(python_code)
    end = f'End-state:\n{self.get_env()}'
    return f'{start}\nResult:\n{result}\n{end}'
anything_from_str_to_dict(data, expected_keys=None, mini_task=lambda x: '')

Versucht, einen String in ein oder mehrere Dictionaries umzuwandeln. Berücksichtigt dabei die erwarteten Schlüssel und ihre Standardwerte.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
def anything_from_str_to_dict(data: str, expected_keys: dict = None, mini_task=lambda x: ''):
    """
    Versucht, einen String in ein oder mehrere Dictionaries umzuwandeln.
    Berücksichtigt dabei die erwarteten Schlüssel und ihre Standardwerte.
    """
    if len(data) < 4:
        return []

    if expected_keys is None:
        expected_keys = {}

    result = []
    json_objects = find_json_objects_in_str(data)
    if not json_objects and data.startswith('[') and data.endswith(']'):
        json_objects = eval(data)
    if json_objects and len(json_objects) > 0 and isinstance(json_objects[0], dict):
        result.extend([{**expected_keys, **ob} for ob in json_objects])
    if not result:
        completed_object = complete_json_object(data, mini_task)
        if completed_object is not None:
            result.append(completed_object)
    if len(result) == 0 and expected_keys:
        result = [{list(expected_keys.keys())[0]: data}]
    for res in result:
        if isinstance(res, list) and len(res) > 0:
            res = res[0]
        for key, value in expected_keys.items():
            if key not in res:
                res[key] = value

    if len(result) == 0:
        fixed = fix_json(data)
        if fixed:
            result.append(fixed)

    return result
complete_json_object(data, mini_task)

Ruft eine Funktion auf, um einen String in das richtige Format zu bringen. Gibt das resultierende JSON-Objekt zurück, wenn die Funktion erfolgreich ist, sonst None.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
def complete_json_object(data: str, mini_task):
    """
    Ruft eine Funktion auf, um einen String in das richtige Format zu bringen.
    Gibt das resultierende JSON-Objekt zurück, wenn die Funktion erfolgreich ist, sonst None.
    """
    ret = mini_task(
        f"Vervollständige das Json Object. Und bringe den string in das Richtige format. data={data}\nJson=")
    if ret:
        return anything_from_str_to_dict(ret)
    return None
detect_shell()

Detects the best available shell and the argument to execute a command. Returns: A tuple of (shell_executable, command_argument). e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
def detect_shell() -> tuple[str, str]:
    """
    Detects the best available shell and the argument to execute a command.
    Returns:
        A tuple of (shell_executable, command_argument).
        e.g., ('/bin/bash', '-c') or ('powershell.exe', '-Command')
    """
    if platform.system() == "Windows":
        if shell_path := shutil.which("pwsh"):
            return shell_path, "-Command"
        if shell_path := shutil.which("powershell"):
            return shell_path, "-Command"
        return "cmd.exe", "/c"

    shell_env = os.environ.get("SHELL")
    if shell_env and shutil.which(shell_env):
        return shell_env, "-c"

    for shell in ["bash", "zsh", "sh"]:
        if shell_path := shutil.which(shell):
            return shell_path, "-c"

    return "/bin/sh", "-c"
extract_text_natively(data, filename='')

Extrahiert Text aus verschiedenen Dateitypen mit nativen Python-Methoden oder reinen Python-Bibliotheken (speziell PyPDF2 für PDFs).

Parameters:

Name Type Description Default
data bytes

Der Inhalt der Datei als Bytes.

required
filename str

Der Originaldateiname, um den Typ zu bestimmen.

''

Returns:

Name Type Description
str str

Der extrahierte Text.

Raises:

Type Description
ValueError

Wenn der Dateityp nicht unterstützt wird oder die Verarbeitung fehlschlägt.

ImportError

Wenn PyPDF2 für die Verarbeitung von PDF-Dateien benötigt, aber nicht installiert ist.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
def extract_text_natively(data: bytes, filename: str = "") -> str:
    """
    Extrahiert Text aus verschiedenen Dateitypen mit nativen Python-Methoden
    oder reinen Python-Bibliotheken (speziell PyPDF2 für PDFs).

    Args:
        data (bytes): Der Inhalt der Datei als Bytes.
        filename (str, optional): Der Originaldateiname, um den Typ zu bestimmen.

    Returns:
        str: Der extrahierte Text.

    Raises:
        ValueError: Wenn der Dateityp nicht unterstützt wird oder die Verarbeitung fehlschlägt.
        ImportError: Wenn PyPDF2 für die Verarbeitung von PDF-Dateien benötigt, aber nicht installiert ist.
    """
    file_ext = filename.lower().split('.')[-1] if '.' in filename else ''

    # 1. DOCX-Verarbeitung (nativ mit zipfile und xml)
    if data.startswith(b'PK\x03\x04'):
        try:
            docx_file = io.BytesIO(data)
            text_parts = []
            with zipfile.ZipFile(docx_file) as zf:
                namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
                body_path = "word/document.xml"
                if body_path in zf.namelist():
                    xml_content = zf.read(body_path)
                    tree = ET.fromstring(xml_content)
                    for para in tree.iter(f"{namespace}p"):
                        texts_in_para = [node.text for node in para.iter(f"{namespace}t") if node.text]
                        if texts_in_para:
                            text_parts.append("".join(texts_in_para))
                return "\n".join(text_parts)
        except (zipfile.BadZipFile, ET.ParseError):
            pass  # Fährt fort, falls es eine ZIP-Datei, aber kein gültiges DOCX ist

    # 2. PDF-Verarbeitung (mit PyPDF2)
    if data.startswith(b'%PDF-'):
        if PyPDF2 is None:
            raise ImportError(
                "Die Bibliothek 'PyPDF2' wird benötigt, um PDF-Dateien zu verarbeiten. Bitte installieren Sie sie mit 'pip install PyPDF2'.")

        try:
            # Erstelle ein In-Memory-Dateiobjekt für PyPDF2
            pdf_file = io.BytesIO(data)
            # Verwende PdfFileReader aus PyPDF2
            pdf_reader = PyPDF2.PdfFileReader(pdf_file)

            text_parts = []
            # Iteriere durch die Seiten
            for page_num in range(pdf_reader.numPages):
                page = pdf_reader.getPage(page_num)
                # Extrahiere Text mit extractText()
                page_text = page.extractText()
                if page_text:
                    text_parts.append(page_text)

            return "\n".join(text_parts)
        except Exception as e:
            raise ValueError(f"PDF-Verarbeitung mit PyPDF2 fehlgeschlagen: {e}")

    # 3. Fallback auf reinen Text (TXT)

    try:
        return data.decode('utf-8')
    except UnicodeDecodeError:
        try:
            return data.decode('latin-1')
        except Exception as e:
            raise ValueError(f"Text-Dekodierung fehlgeschlagen: {e}")
find_json_objects_in_str(data)

Sucht nach JSON-Objekten innerhalb eines Strings. Gibt eine Liste von JSON-Objekten zurück, die im String gefunden wurden.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
1229
1230
1231
1232
1233
1234
1235
1236
1237
def find_json_objects_in_str(data: str):
    """
    Sucht nach JSON-Objekten innerhalb eines Strings.
    Gibt eine Liste von JSON-Objekten zurück, die im String gefunden wurden.
    """
    json_objects = extract_json_objects(data)
    if not isinstance(json_objects, list):
        json_objects = [json_objects]
    return [get_json_from_json_str(ob, 10) for ob in json_objects if get_json_from_json_str(ob, 10) is not None]
get_json_from_json_str(json_str, repeat=1)

Versucht, einen JSON-String in ein Python-Objekt umzuwandeln.

Wenn beim Parsen ein Fehler auftritt, versucht die Funktion, das Problem zu beheben, indem sie das Zeichen an der Position des Fehlers durch ein Escape-Zeichen ersetzt. Dieser Vorgang wird bis zu repeat-mal wiederholt.

Parameters:

Name Type Description Default
json_str str or list or dict

Der JSON-String, der geparst werden soll.

required
repeat int

Die Anzahl der Versuche, das Parsen durchzuführen.

1

Returns:

Type Description
dict or None

Das resultierende Python-Objekt.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
def get_json_from_json_str(json_str: str or list or dict, repeat: int = 1) -> dict or None:
    """Versucht, einen JSON-String in ein Python-Objekt umzuwandeln.

    Wenn beim Parsen ein Fehler auftritt, versucht die Funktion, das Problem zu beheben,
    indem sie das Zeichen an der Position des Fehlers durch ein Escape-Zeichen ersetzt.
    Dieser Vorgang wird bis zu `repeat`-mal wiederholt.

    Args:
        json_str: Der JSON-String, der geparst werden soll.
        repeat: Die Anzahl der Versuche, das Parsen durchzuführen.

    Returns:
        Das resultierende Python-Objekt.
    """
    for _ in range(repeat):
        try:
            return parse_json_with_auto_detection(json_str)
        except json.JSONDecodeError as e:
            unexp = int(re.findall(r'\(char (\d+)\)', str(e))[0])
            unesc = json_str.rfind(r'"', 0, unexp)
            json_str = json_str[:unesc] + r'\"' + json_str[unesc + 1:]
            closg = json_str.find(r'"', unesc + 2)
            json_str = json_str[:closg] + r'\"' + json_str[closg + 1:]
        new = fix_json_object(json_str)
        if new is not None:
            json_str = new
    get_logger().info(f"Unable to parse JSON string after {json_str}")
    return None
parse_json_with_auto_detection(json_data)

Parses JSON data, automatically detecting if a value is a JSON string and parsing it accordingly. If a value cannot be parsed as JSON, it is returned as is.

Source code in toolboxv2/mods/isaa/base/AgentUtils.py
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
def parse_json_with_auto_detection(json_data):
    """
    Parses JSON data, automatically detecting if a value is a JSON string and parsing it accordingly.
    If a value cannot be parsed as JSON, it is returned as is.
    """

    def try_parse_json(value):
        """
        Tries to parse a value as JSON. If the parsing fails, the original value is returned.
        """
        try:
            # print("parse_json_with_auto_detection:", type(value), value)
            parsed_value = json.loads(value)
            # print("parsed_value:", type(parsed_value), parsed_value)
            # If the parsed value is a string, it might be a JSON string, so we try to parse it again
            if isinstance(parsed_value, str):
                return eval(parsed_value)
            else:
                return parsed_value
        except Exception:
            # logging.warning(f"Failed to parse value as JSON: {value}. Exception: {e}")
            return value

    get_logger()

    if isinstance(json_data, dict):
        return {key: parse_json_with_auto_detection(value) for key, value in json_data.items()}
    elif isinstance(json_data, list):
        return [parse_json_with_auto_detection(item) for item in json_data]
    else:
        return try_parse_json(json_data)
KnowledgeBase
Chunk dataclass

Represents a chunk of text with its embedding and metadata

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
27
28
29
30
31
32
33
34
@dataclass(slots=True)
class Chunk:
    """Represents a chunk of text with its embedding and metadata"""
    text: str
    embedding: np.ndarray
    metadata: dict[str, Any]
    content_hash: str
    cluster_id: int | None = None
ConceptAnalysis

Bases: BaseModel

Represents the analysis of key concepts.

Attributes:

Name Type Description
key_concepts list[str]

A list of primary key concepts identified.

relationships list[str]

A list of relationships between the identified key concepts.

importance_hierarchy list[str]

A list that represents the hierarchical importance of the key concepts.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
111
112
113
114
115
116
117
118
119
120
121
122
class ConceptAnalysis(BaseModel):
    """
    Represents the analysis of key concepts.

    Attributes:
        key_concepts (list[str]): A list of primary key concepts identified.
        relationships (list[str]): A list of relationships between the identified key concepts.
        importance_hierarchy (list[str]): A list that represents the hierarchical importance of the key concepts.
    """
    key_concepts: list[str]
    relationships: list[str]
    importance_hierarchy: list[str]
ConceptExtractor

Handles extraction of concepts and relationships from text

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
class ConceptExtractor:
    """Handles extraction of concepts and relationships from text"""

    def __init__(self, knowledge_base, requests_per_second = 85.):
        self.kb = knowledge_base
        self.concept_graph = ConceptGraph()
        self.requests_per_second = requests_per_second

    async def extract_concepts(self, texts: list[str], metadatas: list[dict[str, Any]]) -> list[list[Concept]]:
        """
        Extract concepts from texts using concurrent processing with rate limiting.
        Requests are made at the specified rate while responses are processed asynchronously.
        """
        # Ensure metadatas list matches texts length
        metadatas = metadatas + [{}] * (len(texts) - len(metadatas))

        # Initialize rate limiter
        rate_limiter = DynamicRateLimiter()

        system_prompt = (
            "Analyze the given text and extract key concepts and their relationships. For each concept:\n"
            "1. Identify the concept name and category (technical, domain, method, property, ...)\n"
            "2. Determine relationships with other concepts (uses, part_of, similar_to, depends_on, ...)\n"
            "3. Assess importance (0-1 score) based on centrality to the text\n"
            "4. Extract relevant context snippets\n"
            "5. Max 5 Concepts!\n"
            "only return in json format!\n"
            """{"concepts": [{
                "name": "concept_name",
                "category": "category_name",
                "relationships": {
                    "relationship_type": ["related_concept1", "related_concept2"]
                },
                "importance_score": 0.0,
                "context_snippets": ["relevant text snippet"]
            }]}\n"""
        )

        # Prepare all requests
        requests = [
            (idx, f"Text to Convert in to JSON structure:\n{text}", system_prompt, metadata)
            for idx, (text, metadata) in enumerate(zip(texts, metadatas, strict=False))
        ]

        async def process_single_request(idx: int, prompt: str, system_prompt: str, metadata: dict[str, Any]):
            """Process a single request with rate limiting"""
            try:
                from toolboxv2.mods.isaa.extras.adapter import litellm_complete
                # Wait for rate limit
                await rate_limiter.acquire()
                i__[1] += 1
                # Make API call without awaiting the response
                response_future = litellm_complete(
                    prompt=prompt,
                    system_prompt=system_prompt,
                    response_format=Concepts,
                    model_name=self.kb.model_name,
                    fallbacks=["groq/gemma2-9b-it"] +
                              [m for m in os.getenv("FALLBACKS_MODELS_PREM", '').split(',') if m]
                )

                return idx, response_future

            except Exception as e:
                print(f"Error initiating request {idx}: {str(e)}")
                return idx, None

        async def process_response(idx: int, response_future) -> list[Concept]:
            """Process the response once it's ready"""
            try:
                if response_future is None:
                    return []

                response = await response_future
                return await self._process_response(response, metadatas[idx])

            except Exception as e:
                print(f"Error processing response {idx}: {str(e)}")
                return []

        # Create tasks for all requests
        request_tasks = []
        batch_size = self.kb.batch_size

        rate_limiter.update_rate(self.requests_per_second)

        for batch_start in range(0, len(requests), batch_size):
            batch = requests[batch_start:batch_start + batch_size]

            # Create tasks for the batch
            batch_tasks = [
                process_single_request(idx, prompt, sys_prompt, meta)
                for idx, prompt, sys_prompt, meta in batch
            ]
            request_tasks.extend(batch_tasks)

        # Execute all requests with rate limiting
        request_results = await asyncio.gather(*request_tasks)

        # Process responses as they complete
        response_tasks = [
            process_response(idx, response_future)
            for idx, response_future in request_results
        ]

        # Gather all results
        all_results = await asyncio.gather(*response_tasks)

        # Sort results by original index
        sorted_results = [[] for _ in texts]
        for idx, concepts in enumerate(all_results):
            sorted_results[idx] = concepts

        return sorted_results

    async def _process_response(self, response: Any, metadata: dict[str, Any]) -> list[Concept]:
        """Helper method to process a single response and convert it to Concepts"""
        try:
            # Extract content from response
            if hasattr(response, 'choices'):
                content = response.choices[0].message.content
                if content is None:
                    content = response.choices[0].message.tool_calls[0].function.arguments
                if content is None:
                    return []
            elif isinstance(response, str):
                content = response
            else:
                print(f"Unexpected response type: {type(response)}")
                return []

            from toolboxv2.mods.isaa.extras.filter import after_format
            # Parse JSON and create concepts
            concept_data = after_format(content)
            concepts = []

            for concept_info in concept_data.get("concepts", []):
                concept = Concept(
                    name=concept_info["name"],
                    category=concept_info.get("category", "N/A"),
                    relationships={k: set(v) for k, v in concept_info.get("relationships", {}).items()},
                    importance_score=concept_info.get("importance_score", 0.1),
                    context_snippets=concept_info.get("context_snippets", "N/A"),
                    metadata=metadata
                )
                concepts.append(concept)
                self.concept_graph.add_concept(concept)

            return concepts

        except Exception:
            i__[2] +=1
            return []

    async def process_chunks(self, chunks: list[Chunk]) -> None:
        """
        Process all chunks in batch to extract and store concepts.
        Each chunk's metadata will be updated with the concept names and relationships.
        """
        # Gather all texts from the chunks.
        texts = [chunk.text for chunk in chunks]
        # Call extract_concepts once with all texts.
        all_concepts = await self.extract_concepts(texts, [chunk.metadata for chunk in chunks])

        # Update each chunk's metadata with its corresponding concepts.
        for chunk, concepts in zip(chunks, all_concepts, strict=False):
            chunk.metadata["concepts"] = [c.name for c in concepts]
            chunk.metadata["concept_relationships"] = {
                c.name: {k: list(v) for k, v in c.relationships.items()}
                for c in concepts
            }

    async def query_concepts(self, query: str) -> dict[str, any]:
        """Query the concept graph based on natural language query"""

        system_prompt = """
        Convert the natural language query about concepts into a structured format that specifies:
        1. Main concepts of interest
        2. Desired relationship types
        3. Any category filters
        4. Importance threshold

        Format as JSON.
        """

        prompt = f"""
        Query: {query}

        Convert to this JSON structure:
        {{
            "target_concepts": ["concept1", "concept2"],
            "relationship_types": ["type1", "type2"],
            "categories": ["category1", "category2"],
            "min_importance": 0.0
        }}
        """

        try:
            from toolboxv2.mods.isaa.extras.adapter import litellm_complete
            response = await litellm_complete(
                model_name=self.kb.model_name,
                prompt=prompt,
                system_prompt=system_prompt,
                response_format=TConcept
            )

            query_params = json.loads(response)

            results = {
                "concepts": {},
                "relationships": [],
                "groups": []
            }

            # Find matching concepts
            for concept_name in query_params["target_concepts"]:
                if concept_name in self.concept_graph.concepts:
                    concept = self.concept_graph.concepts[concept_name]
                    if concept.importance_score >= query_params["min_importance"]:
                        results["concepts"][concept_name] = {
                            "category": concept.category,
                            "importance": concept.importance_score,
                            "context": concept.context_snippets
                        }

                        # Get relationships
                        for rel_type in query_params["relationship_types"]:
                            related = self.concept_graph.get_related_concepts(
                                concept_name, rel_type
                            )
                            for related_concept in related:
                                results["relationships"].append({
                                    "from": concept_name,
                                    "to": related_concept,
                                    "type": rel_type
                                })

            # Group concepts by category
            category_groups = defaultdict(list)
            for concept_name, concept_info in results["concepts"].items():
                category_groups[concept_info["category"]].append(concept_name)
            results["groups"] = [
                {"category": cat, "concepts": concepts}
                for cat, concepts in category_groups.items()
            ]

            return results

        except Exception as e:
            print(f"Error querying concepts: {str(e)}")
            return {"concepts": {}, "relationships": [], "groups": []}
extract_concepts(texts, metadatas) async

Extract concepts from texts using concurrent processing with rate limiting. Requests are made at the specified rate while responses are processed asynchronously.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
async def extract_concepts(self, texts: list[str], metadatas: list[dict[str, Any]]) -> list[list[Concept]]:
    """
    Extract concepts from texts using concurrent processing with rate limiting.
    Requests are made at the specified rate while responses are processed asynchronously.
    """
    # Ensure metadatas list matches texts length
    metadatas = metadatas + [{}] * (len(texts) - len(metadatas))

    # Initialize rate limiter
    rate_limiter = DynamicRateLimiter()

    system_prompt = (
        "Analyze the given text and extract key concepts and their relationships. For each concept:\n"
        "1. Identify the concept name and category (technical, domain, method, property, ...)\n"
        "2. Determine relationships with other concepts (uses, part_of, similar_to, depends_on, ...)\n"
        "3. Assess importance (0-1 score) based on centrality to the text\n"
        "4. Extract relevant context snippets\n"
        "5. Max 5 Concepts!\n"
        "only return in json format!\n"
        """{"concepts": [{
            "name": "concept_name",
            "category": "category_name",
            "relationships": {
                "relationship_type": ["related_concept1", "related_concept2"]
            },
            "importance_score": 0.0,
            "context_snippets": ["relevant text snippet"]
        }]}\n"""
    )

    # Prepare all requests
    requests = [
        (idx, f"Text to Convert in to JSON structure:\n{text}", system_prompt, metadata)
        for idx, (text, metadata) in enumerate(zip(texts, metadatas, strict=False))
    ]

    async def process_single_request(idx: int, prompt: str, system_prompt: str, metadata: dict[str, Any]):
        """Process a single request with rate limiting"""
        try:
            from toolboxv2.mods.isaa.extras.adapter import litellm_complete
            # Wait for rate limit
            await rate_limiter.acquire()
            i__[1] += 1
            # Make API call without awaiting the response
            response_future = litellm_complete(
                prompt=prompt,
                system_prompt=system_prompt,
                response_format=Concepts,
                model_name=self.kb.model_name,
                fallbacks=["groq/gemma2-9b-it"] +
                          [m for m in os.getenv("FALLBACKS_MODELS_PREM", '').split(',') if m]
            )

            return idx, response_future

        except Exception as e:
            print(f"Error initiating request {idx}: {str(e)}")
            return idx, None

    async def process_response(idx: int, response_future) -> list[Concept]:
        """Process the response once it's ready"""
        try:
            if response_future is None:
                return []

            response = await response_future
            return await self._process_response(response, metadatas[idx])

        except Exception as e:
            print(f"Error processing response {idx}: {str(e)}")
            return []

    # Create tasks for all requests
    request_tasks = []
    batch_size = self.kb.batch_size

    rate_limiter.update_rate(self.requests_per_second)

    for batch_start in range(0, len(requests), batch_size):
        batch = requests[batch_start:batch_start + batch_size]

        # Create tasks for the batch
        batch_tasks = [
            process_single_request(idx, prompt, sys_prompt, meta)
            for idx, prompt, sys_prompt, meta in batch
        ]
        request_tasks.extend(batch_tasks)

    # Execute all requests with rate limiting
    request_results = await asyncio.gather(*request_tasks)

    # Process responses as they complete
    response_tasks = [
        process_response(idx, response_future)
        for idx, response_future in request_results
    ]

    # Gather all results
    all_results = await asyncio.gather(*response_tasks)

    # Sort results by original index
    sorted_results = [[] for _ in texts]
    for idx, concepts in enumerate(all_results):
        sorted_results[idx] = concepts

    return sorted_results
process_chunks(chunks) async

Process all chunks in batch to extract and store concepts. Each chunk's metadata will be updated with the concept names and relationships.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
async def process_chunks(self, chunks: list[Chunk]) -> None:
    """
    Process all chunks in batch to extract and store concepts.
    Each chunk's metadata will be updated with the concept names and relationships.
    """
    # Gather all texts from the chunks.
    texts = [chunk.text for chunk in chunks]
    # Call extract_concepts once with all texts.
    all_concepts = await self.extract_concepts(texts, [chunk.metadata for chunk in chunks])

    # Update each chunk's metadata with its corresponding concepts.
    for chunk, concepts in zip(chunks, all_concepts, strict=False):
        chunk.metadata["concepts"] = [c.name for c in concepts]
        chunk.metadata["concept_relationships"] = {
            c.name: {k: list(v) for k, v in c.relationships.items()}
            for c in concepts
        }
query_concepts(query) async

Query the concept graph based on natural language query

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
async def query_concepts(self, query: str) -> dict[str, any]:
    """Query the concept graph based on natural language query"""

    system_prompt = """
    Convert the natural language query about concepts into a structured format that specifies:
    1. Main concepts of interest
    2. Desired relationship types
    3. Any category filters
    4. Importance threshold

    Format as JSON.
    """

    prompt = f"""
    Query: {query}

    Convert to this JSON structure:
    {{
        "target_concepts": ["concept1", "concept2"],
        "relationship_types": ["type1", "type2"],
        "categories": ["category1", "category2"],
        "min_importance": 0.0
    }}
    """

    try:
        from toolboxv2.mods.isaa.extras.adapter import litellm_complete
        response = await litellm_complete(
            model_name=self.kb.model_name,
            prompt=prompt,
            system_prompt=system_prompt,
            response_format=TConcept
        )

        query_params = json.loads(response)

        results = {
            "concepts": {},
            "relationships": [],
            "groups": []
        }

        # Find matching concepts
        for concept_name in query_params["target_concepts"]:
            if concept_name in self.concept_graph.concepts:
                concept = self.concept_graph.concepts[concept_name]
                if concept.importance_score >= query_params["min_importance"]:
                    results["concepts"][concept_name] = {
                        "category": concept.category,
                        "importance": concept.importance_score,
                        "context": concept.context_snippets
                    }

                    # Get relationships
                    for rel_type in query_params["relationship_types"]:
                        related = self.concept_graph.get_related_concepts(
                            concept_name, rel_type
                        )
                        for related_concept in related:
                            results["relationships"].append({
                                "from": concept_name,
                                "to": related_concept,
                                "type": rel_type
                            })

        # Group concepts by category
        category_groups = defaultdict(list)
        for concept_name, concept_info in results["concepts"].items():
            category_groups[concept_info["category"]].append(concept_name)
        results["groups"] = [
            {"category": cat, "concepts": concepts}
            for cat, concepts in category_groups.items()
        ]

        return results

    except Exception as e:
        print(f"Error querying concepts: {str(e)}")
        return {"concepts": {}, "relationships": [], "groups": []}
ConceptGraph

Manages concept relationships and hierarchies

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
class ConceptGraph:
    """Manages concept relationships and hierarchies"""

    def __init__(self):
        self.concepts: dict[str, Concept] = {}

    def add_concept(self, concept: Concept):
        """Add or update a concept in the graph"""
        if concept.name.lower() in self.concepts:
            # Merge relationships and context
            existing = self.concepts[concept.name.lower()]
            for rel_type, related in concept.relationships.items():
                if rel_type not in existing.relationships:
                    existing.relationships[rel_type] = set()
                existing.relationships[rel_type].update(related)
            existing.context_snippets.extend(concept.context_snippets)
            # Update importance score with rolling average
            existing.importance_score = (existing.importance_score + concept.importance_score) / 2
        else:
            self.concepts[concept.name.lower()] = concept

    def get_related_concepts(self, concept_name: str, relationship_type: str | None = None) -> set[str]:
        """Get related concepts, optionally filtered by relationship type"""
        if concept_name not in self.concepts:
            return set()

        concept = self.concepts[concept_name.lower()]
        if relationship_type:
            return concept.relationships.get(relationship_type, set())

        related = set()
        for relations in concept.relationships.values():
            related.update(relations)
        return related


    def convert_to_networkx(self) -> nx.DiGraph:
        """Convert ConceptGraph to NetworkX graph with layout"""
        print(f"Converting to NetworkX graph with {len(self.concepts.values())} concepts")

        G = nx.DiGraph()

        if len(self.concepts.values()) == 0:
            return G

        for concept in self.concepts.values():
            cks = '\n - '.join(concept.context_snippets[:4])
            G.add_node(
                concept.name,
                size=concept.importance_score * 10,
                group=concept.category,
                title=f"""
                    {concept.name}
                    Category: {concept.category}
                    Importance: {concept.importance_score:.2f}
                    Context: \n - {cks}
                    """
            )

            for rel_type, targets in concept.relationships.items():
                for target in targets:
                    G.add_edge(concept.name, target, label=rel_type, title=rel_type)

        return G
add_concept(concept)

Add or update a concept in the graph

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
174
175
176
177
178
179
180
181
182
183
184
185
186
187
def add_concept(self, concept: Concept):
    """Add or update a concept in the graph"""
    if concept.name.lower() in self.concepts:
        # Merge relationships and context
        existing = self.concepts[concept.name.lower()]
        for rel_type, related in concept.relationships.items():
            if rel_type not in existing.relationships:
                existing.relationships[rel_type] = set()
            existing.relationships[rel_type].update(related)
        existing.context_snippets.extend(concept.context_snippets)
        # Update importance score with rolling average
        existing.importance_score = (existing.importance_score + concept.importance_score) / 2
    else:
        self.concepts[concept.name.lower()] = concept
convert_to_networkx()

Convert ConceptGraph to NetworkX graph with layout

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
def convert_to_networkx(self) -> nx.DiGraph:
    """Convert ConceptGraph to NetworkX graph with layout"""
    print(f"Converting to NetworkX graph with {len(self.concepts.values())} concepts")

    G = nx.DiGraph()

    if len(self.concepts.values()) == 0:
        return G

    for concept in self.concepts.values():
        cks = '\n - '.join(concept.context_snippets[:4])
        G.add_node(
            concept.name,
            size=concept.importance_score * 10,
            group=concept.category,
            title=f"""
                {concept.name}
                Category: {concept.category}
                Importance: {concept.importance_score:.2f}
                Context: \n - {cks}
                """
        )

        for rel_type, targets in concept.relationships.items():
            for target in targets:
                G.add_edge(concept.name, target, label=rel_type, title=rel_type)

    return G
get_related_concepts(concept_name, relationship_type=None)

Get related concepts, optionally filtered by relationship type

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
189
190
191
192
193
194
195
196
197
198
199
200
201
def get_related_concepts(self, concept_name: str, relationship_type: str | None = None) -> set[str]:
    """Get related concepts, optionally filtered by relationship type"""
    if concept_name not in self.concepts:
        return set()

    concept = self.concepts[concept_name.lower()]
    if relationship_type:
        return concept.relationships.get(relationship_type, set())

    related = set()
    for relations in concept.relationships.values():
        related.update(relations)
    return related
Concepts

Bases: BaseModel

Represents a collection of key concepts.

Attributes:

Name Type Description
concepts List[rConcept]

A list of Concept instances, each representing an individual key concept.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
102
103
104
105
106
107
108
109
class Concepts(BaseModel):
    """
    Represents a collection of key concepts.

    Attributes:
        concepts (List[rConcept]): A list of Concept instances, each representing an individual key concept.
    """
    concepts: list[rConcept]
DataModel

Bases: BaseModel

The main data model that encapsulates the overall analysis.

Attributes:

Name Type Description
main_summary str

A Detailed overview summarizing the key findings and relations format MD string.

concept_analysis ConceptAnalysis

An instance containing the analysis of key concepts.

topic_insights TopicInsights

An instance containing insights regarding the topics.

relevance_assessment RelevanceAssessment

An instance assessing the relevance and alignment of the query.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
153
154
155
156
157
158
159
160
161
162
163
164
165
166
class DataModel(BaseModel):
    """
    The main data model that encapsulates the overall analysis.

    Attributes:
        main_summary (str): A Detailed overview summarizing the key findings and relations format MD string.
        concept_analysis (ConceptAnalysis): An instance containing the analysis of key concepts.
        topic_insights (TopicInsights): An instance containing insights regarding the topics.
        relevance_assessment (RelevanceAssessment): An instance assessing the relevance and alignment of the query.
    """
    main_summary: str
    concept_analysis: ConceptAnalysis
    topic_insights: TopicInsights
    relevance_assessment: RelevanceAssessment
DynamicRateLimiter
Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
class DynamicRateLimiter:
    def __init__(self):
        self.last_request_time = 0.0
        self._lock = asyncio.Lock()

    def update_rate(self, requests_per_second: float):
        """Update rate limit dynamically"""
        self.min_interval = 1.0 / requests_per_second if requests_per_second > 0 else float('inf')

    async def acquire(self):
        """Acquire permission to make a request"""
        async with self._lock:
            current_time = time.time()
            time_since_last = current_time - self.last_request_time
            if time_since_last < self.min_interval:
                wait_time = self.min_interval - time_since_last
                await asyncio.sleep(wait_time)
            self.last_request_time = time.time()
acquire() async

Acquire permission to make a request

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
266
267
268
269
270
271
272
273
274
async def acquire(self):
    """Acquire permission to make a request"""
    async with self._lock:
        current_time = time.time()
        time_since_last = current_time - self.last_request_time
        if time_since_last < self.min_interval:
            wait_time = self.min_interval - time_since_last
            await asyncio.sleep(wait_time)
        self.last_request_time = time.time()
update_rate(requests_per_second)

Update rate limit dynamically

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
262
263
264
def update_rate(self, requests_per_second: float):
    """Update rate limit dynamically"""
    self.min_interval = 1.0 / requests_per_second if requests_per_second > 0 else float('inf')
GraphVisualizer
Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
class GraphVisualizer:
    @staticmethod
    def visualize(nx_graph: nx.DiGraph, output_file: str = "concept_graph.html", get_output=False):
        """Create interactive visualization using PyVis"""
        from pyvis.network import Network
        net = Network(
            height="800px",
            width="100%",
            notebook=False,
            directed=True,
            bgcolor="#1a1a1a",
            font_color="white"
        )

        net.from_nx(nx_graph)

        net.save_graph(output_file)
        print(f"Graph saved to {output_file} Open in browser to view.", len(nx_graph))
        if get_output:
            c = open(output_file, encoding="utf-8").read()
            os.remove(output_file)
            return c
visualize(nx_graph, output_file='concept_graph.html', get_output=False) staticmethod

Create interactive visualization using PyVis

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
@staticmethod
def visualize(nx_graph: nx.DiGraph, output_file: str = "concept_graph.html", get_output=False):
    """Create interactive visualization using PyVis"""
    from pyvis.network import Network
    net = Network(
        height="800px",
        width="100%",
        notebook=False,
        directed=True,
        bgcolor="#1a1a1a",
        font_color="white"
    )

    net.from_nx(nx_graph)

    net.save_graph(output_file)
    print(f"Graph saved to {output_file} Open in browser to view.", len(nx_graph))
    if get_output:
        c = open(output_file, encoding="utf-8").read()
        os.remove(output_file)
        return c
KnowledgeBase
Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
class KnowledgeBase:
    def __init__(self, embedding_dim: int = 256, similarity_threshold: float = 0.61, batch_size: int = 12,
                 n_clusters: int = 4, deduplication_threshold: float = 0.85, model_name=os.getenv("SUMMARYMODEL"),
                 embedding_model=os.getenv("DEFAULTMODELEMBEDDING"),
                 vis_class:str | None = "FaissVectorStore",
                 vis_kwargs:dict[str, Any] | None=None,
                 requests_per_second=85.,
                 chunk_size: int = 3600,
                 chunk_overlap: int = 130,
                 separator: str = "\n"
                 ):
        """Initialize the knowledge base with given parameters"""

        self.existing_hashes: set[str] = set()
        self.embedding_model = embedding_model
        self.embedding_dim = embedding_dim
        self.similarity_threshold = similarity_threshold
        self.deduplication_threshold = deduplication_threshold
        if model_name == "openrouter/mistralai/mistral-nemo":
            batch_size = 9
            requests_per_second = 1.5
        self.batch_size = batch_size
        self.n_clusters = n_clusters
        self.model_name = model_name
        self.sto: list = []

        self.text_splitter = TextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap, separator=separator)
        self.similarity_graph = {}
        self.concept_extractor = ConceptExtractor(self, requests_per_second)

        self.vis_class = None
        self.vis_kwargs = None
        self.vdb = None
        self.init_vis(vis_class, vis_kwargs)

    def init_vis(self, vis_class, vis_kwargs):
        if vis_class is None:
            vis_class = "FaissVectorStore"
        if vis_class == "FaissVectorStore":
            if vis_kwargs is None:
                vis_kwargs = {
                    "dimension": self.embedding_dim
                }
            self.vdb = FaissVectorStore(**vis_kwargs)
        else:
            from toolboxv2.mods.isaa.base.VectorStores.taichiNumpyNumbaVectorStores import (
                EnhancedVectorStore,
                FastVectorStore1,
                FastVectorStoreO,
                NumpyVectorStore,
                VectorStoreConfig,
            )
        if vis_class == "FastVectorStoreO":
            if vis_kwargs is None:
                vis_kwargs = {
                    "embedding_size": self.embedding_dim
                }
            self.vdb = FastVectorStoreO(**vis_kwargs)
        if vis_class == "EnhancedVectorStore":
            if vis_kwargs is None:
                vis_kwargs = {
                    "dimension": self.embedding_dim
                }
            vis_kwargs = VectorStoreConfig(**vis_kwargs)
            self.vdb = EnhancedVectorStore(vis_kwargs)
        if vis_class == "FastVectorStore1":
            self.vdb = FastVectorStore1()
        if vis_class == "NumpyVectorStore":
            self.vdb = NumpyVectorStore()

        self.vis_class = vis_class
        self.vis_kwargs = vis_kwargs


    @staticmethod
    def compute_hash(text: str) -> str:
        """Compute SHA-256 hash of text"""
        return hashlib.sha256(text.encode('utf-8', errors='ignore')).hexdigest()

    async def _get_embeddings(self, texts: list[str]) -> np.ndarray:
        """Get normalized embeddings in batches"""
        try:
            async def process_batch(batch: list[str]) -> np.ndarray:
                from toolboxv2.mods.isaa.extras.adapter import litellm_embed
                # print("Processing", batch)
                embeddings = await litellm_embed(texts=batch, model=self.embedding_model, dimensions=self.embedding_dim)
                return normalize_vectors(embeddings)

            tasks = []
            for i in range(0, len(texts), self.batch_size):
                batch = texts[i:i + self.batch_size]
                tasks.append(process_batch(batch))

            embeddings = await asyncio.gather(*tasks)
            i__[0] += len(texts)
            return np.vstack(embeddings)
        except Exception as e:
            get_logger().error(f"Error generating embeddings: {str(e)}")
            raise



    def _remove_similar_chunks(self, threshold: float = None) -> int:
        """Remove chunks that are too similar to each other"""
        if len(self.vdb.chunks) < 2:
            return 0

        if threshold is None:
            threshold = self.deduplication_threshold

        try:
            # Get all embeddings
            embeddings = np.vstack([c.embedding for c in self.vdb.chunks])
            n = len(embeddings)

            # Compute similarity matrix
            similarities = np.dot(embeddings, embeddings.T)

            # Create mask for chunks to keep
            keep_mask = np.ones(n, dtype=bool)

            # Iterate through chunks
            for i in range(n):
                if not keep_mask[i]:
                    continue

                # Find chunks that are too similar to current chunk
                similar_indices = similarities[i] >= threshold
                similar_indices[i] = False  # Don't count self-similarity

                # Mark similar chunks for removal
                keep_mask[similar_indices] = False

            # Keep only unique chunks
            unique_chunks = [chunk for chunk, keep in zip(self.vdb.chunks, keep_mask, strict=False) if keep]
            removed_count = len(self.vdb.chunks) - len(unique_chunks)

            # Update chunks and hashes
            self.vdb.chunks = unique_chunks
            self.existing_hashes = {chunk.content_hash for chunk in self.vdb.chunks}

            # Rebuild index if chunks were removed
            if removed_count > 0:
                self.vdb.rebuild_index()


            return removed_count

        except Exception as e:
            get_logger().error(f"Error removing similar chunks: {str(e)}")
            raise

    async def _add_data(
        self,
        texts: list[str],
        metadata: list[dict[str, Any]] | None= None,
    ) -> tuple[int, int]:
        """
        Process and add new data to the knowledge base
        Returns: Tuple of (added_count, duplicate_count)
        """
        if len(texts) == 0:
            return -1, -1
        try:
            # Compute hashes and filter exact duplicates
            hashes = [self.compute_hash(text) for text in texts]
            unique_data = []
            for t, m, h in zip(texts, metadata, hashes, strict=False):
                if h in self.existing_hashes:
                    continue
                # Update existing hashes
                self.existing_hashes.add(h)
                unique_data.append((t, m, h))

            if not unique_data:
                return 0, len(texts)

            # Get embeddings
            embeddings = await self._get_embeddings(texts)

            texts = []
            metadata = []
            hashes = []
            embeddings_final = []
            if len(self.vdb.chunks):
                for i, d in enumerate(unique_data):
                    c = self.vdb.search(embeddings[i], 5, self.deduplication_threshold)
                    if len(c) > 2:
                        continue
                    t, m, h = d
                    texts.append(t)
                    metadata.append(m)
                    hashes.append(h)
                    embeddings_final.append(embeddings[i])

            else:
                texts , metadata, hashes = zip(*unique_data, strict=False)
                embeddings_final = embeddings

            if not texts:  # All were similar to existing chunks
                return 0, len(unique_data)

            # Create and add new chunks
            new_chunks = [
                Chunk(text=t, embedding=e, metadata=m, content_hash=h)
                for t, e, m, h in zip(texts, embeddings_final, metadata, hashes, strict=False)
            ]

            # Add new chunks
            # Update index
            if new_chunks:
                all_embeddings = np.vstack([c.embedding for c in new_chunks])
                self.vdb.add_embeddings(all_embeddings, new_chunks)

            # Remove similar chunks from the entire collection
            removed = self._remove_similar_chunks()
            get_logger().info(f"Removed {removed} similar chunks during deduplication")
            # Invalidate visualization cache

            if len(new_chunks) - removed > 0:
                # Process new chunks for concepts
                await self.concept_extractor.process_chunks(new_chunks)
            print("[total, calls, errors]", i__)

            return len(new_chunks) - removed, len(texts) - len(new_chunks) + removed

        except Exception as e:
            get_logger().error(f"Error adding data: {str(e)}")
            raise


    async def add_data(
        self,
        texts: list[str],
        metadata: list[dict[str, Any]] | None = None, direct:bool = False
    ) -> tuple[int, int]:
        """Enhanced version with smart splitting and clustering"""
        if isinstance(texts, str):
            texts = [texts]
        if metadata is None:
            metadata = [{}] * len(texts)
        if isinstance(metadata, dict):
            metadata = [metadata]
        if len(texts) != len(metadata):
            raise ValueError("Length of texts and metadata must match")

        if not direct and len(texts) == 1 and len(texts[0]) < 10_000:
            if len(self.sto) < self.batch_size and len(texts) == 1:
                self.sto.append((texts[0], metadata[0]))
                return -1, -1
            if len(self.sto) >= self.batch_size:
                _ = [texts.append(t) or metadata.append([m]) for (t, m) in self.sto]
                self.sto = []

        # Split large texts
        split_texts = []
        split_metadata = []

        while Spinner("Saving Data to Memory", symbols='t'):

            for idx, text in enumerate(texts):
                chunks = self.text_splitter.split_text(text)
                split_texts.extend(chunks)

                # Adjust metadata for splits
                meta = metadata[idx] if metadata else {}
                if isinstance(meta, list):
                    meta = meta[0]
                for i, _chunk in enumerate(chunks):
                    chunk_meta = meta.copy()
                    chunk_meta.update({
                        'chunk_index': i,
                        'total_chunks': len(chunks),
                        'original_text_id': idx
                    })
                    split_metadata.append(chunk_meta)

            return await self._add_data(split_texts, split_metadata)

    def _update_similarity_graph(self, embeddings: np.ndarray, chunk_ids: list[int]):
        """Update similarity graph for connected information detection"""
        similarities = np.dot(embeddings, embeddings.T)

        for i in range(len(chunk_ids)):
            for j in range(i + 1, len(chunk_ids)):
                if similarities[i, j] >= self.similarity_threshold:
                    id1, id2 = chunk_ids[i], chunk_ids[j]
                    if id1 not in self.similarity_graph:
                        self.similarity_graph[id1] = set()
                    if id2 not in self.similarity_graph:
                        self.similarity_graph[id2] = set()
                    self.similarity_graph[id1].add(id2)
                    self.similarity_graph[id2].add(id1)

    async def retrieve(
        self,
        query: str="",
        query_embedding: np.ndarray | None = None,
        k: int = 5,
        min_similarity: float = 0.2,
        include_connected: bool = True
    ) -> list[Chunk]:
        """Enhanced retrieval with connected information"""
        if query_embedding is None:
            query_embedding = (await self._get_embeddings([query]))[0]
        k = min(k, len(self.vdb.chunks))
        if k <= 0:
            return []
        initial_results = self.vdb.search(query_embedding, k, min_similarity)

        if not include_connected or not initial_results:
            return initial_results

        # Find connected chunks
        connected_chunks = set()
        for chunk in initial_results:
            chunk_id = self.vdb.chunks.index(chunk)
            if chunk_id in self.similarity_graph:
                connected_chunks.update(self.similarity_graph[chunk_id])

        # Add connected chunks to results
        all_chunks = self.vdb.chunks
        additional_results = [all_chunks[i] for i in connected_chunks
                              if all_chunks[i] not in initial_results]

        # Sort by similarity to query
        all_results = initial_results + additional_results

        return sorted(
            all_results,
            key=lambda x: np.dot(x.embedding, query_embedding),
            reverse=True
        )[:k * 2]  # Return more results when including connected information

    async def forget_irrelevant(self, irrelevant_concepts: list[str], similarity_threshold: float | None=None) -> int:
        """
        Remove chunks similar to irrelevant concepts
        Returns: Number of chunks removed
        """
        if not irrelevant_concepts:
            return 0

        if similarity_threshold is None:
            similarity_threshold = self.similarity_threshold

        try:
            irrelevant_embeddings = await self._get_embeddings(irrelevant_concepts)
            initial_count = len(self.vdb.chunks)

            def is_relevant(chunk: Chunk) -> bool:
                similarities = np.dot(chunk.embedding, irrelevant_embeddings.T)
                do_keep = np.max(similarities) < similarity_threshold
                if do_keep:
                    return True
                for c in chunk.metadata.get("concepts", []):
                    if c in self.concept_extractor.concept_graph.concepts:
                        del self.concept_extractor.concept_graph.concepts[c]
                return False

            relevant_chunks = [chunk for chunk in self.vdb.chunks if is_relevant(chunk)]
            self.vdb.chunks = relevant_chunks
            self.existing_hashes = {chunk.content_hash for chunk in self.vdb.chunks}
            self.vdb.rebuild_index()

            return initial_count - len(self.vdb.chunks)

        except Exception as e:
            get_logger().error(f"Error forgetting irrelevant concepts: {str(e)}")
            raise

    ## ----------------------------------------------------------------

    def _cluster_chunks(
        self,
        chunks: list[Chunk],
        query_embedding: np.ndarray | None = None,
        min_cluster_size: int = 2,
        min_samples: int = 1,
        max_clusters: int = 10
    ) -> dict[int, list[Chunk]]:
        """
        Enhanced clustering of chunks into topics with query awareness
        and dynamic parameter adjustment
        """
        if len(chunks) < 2:
            return {0: chunks}

        embeddings = np.vstack([chunk.embedding for chunk in chunks])

        # Normalize embeddings for cosine similarity
        embeddings = normalize_vectors(embeddings)

        # If query is provided, weight embeddings by query relevance
        if query_embedding is not None:
            query_similarities = np.dot(embeddings, query_embedding)
            # Apply soft weighting to maintain structure while considering query relevance
            embeddings = embeddings * query_similarities[:, np.newaxis]
            embeddings = normalize_vectors(embeddings)

        # Dynamic parameter adjustment based on dataset size
        adjusted_min_cluster_size = max(
            min_cluster_size,
            min(len(chunks) // 10, 5)  # Scale with data size, max 5
        )

        adjusted_min_samples = max(
            min_samples,
            adjusted_min_cluster_size // 2
        )

        # Try different parameter combinations for optimal clustering
        best_clusters = None
        best_score = float('-inf')

        epsilon_range = [0.2, 0.3, 0.4]
        try:
            HDBSCAN = __import__('sklearn.cluster').HDBSCAN
        except:
            print("install scikit-learn pip install scikit-learn for better results")
            return self._fallback_clustering(chunks, query_embedding)

        for epsilon in epsilon_range:
            clusterer = HDBSCAN(
                min_cluster_size=adjusted_min_cluster_size,
                min_samples=adjusted_min_samples,
                metric='cosine',
                cluster_selection_epsilon=epsilon
            )

            cluster_labels = clusterer.fit_predict(embeddings)

            # Skip if all points are noise
            if len(set(cluster_labels)) <= 1:
                continue

            # Calculate clustering quality metrics
            score = self._evaluate_clustering(
                embeddings,
                cluster_labels,
                query_embedding
            )

            if score > best_score:
                best_score = score
                best_clusters = cluster_labels

        # If no good clustering found, fall back to simpler approach
        if best_clusters is None:
            return self._fallback_clustering(chunks, query_embedding)

        # Organize chunks by cluster
        clusters: dict[int, list[Chunk]] = {}

        # Sort clusters by size and relevance
        cluster_scores = []

        for label in set(best_clusters):
            if label == -1:  # Handle noise points separately
                continue

            # Fixed: Use boolean mask to select chunks for current cluster
            cluster_mask = best_clusters == label
            cluster_chunks = [chunk for chunk, is_in_cluster in zip(chunks, cluster_mask, strict=False) if is_in_cluster]

            # Skip empty clusters
            if not cluster_chunks:
                continue

            # Calculate cluster score based on size and query relevance
            score = len(cluster_chunks)
            if query_embedding is not None:
                cluster_embeddings = np.vstack([c.embedding for c in cluster_chunks])
                query_relevance = np.mean(np.dot(cluster_embeddings, query_embedding))
                score = score * (1 + query_relevance)  # Boost by relevance

            cluster_scores.append((label, score, cluster_chunks))

        # Sort clusters by score and limit to max_clusters
        cluster_scores.sort(key=lambda x: x[1], reverse=True)

        # Assign cleaned clusters
        for i, (_, _, cluster_chunks) in enumerate(cluster_scores[:max_clusters]):
            clusters[i] = cluster_chunks

        # Handle noise points by assigning to nearest cluster
        noise_chunks = [chunk for chunk, label in zip(chunks, best_clusters, strict=False) if label == -1]
        if noise_chunks:
            self._assign_noise_points(noise_chunks, clusters, query_embedding)

        return clusters

    @staticmethod
    def _evaluate_clustering(
        embeddings: np.ndarray,
        labels: np.ndarray,
        query_embedding: np.ndarray | None = None
    ) -> float:
        """
        Evaluate clustering quality using multiple metrics
        """
        if len(set(labels)) <= 1:
            return float('-inf')

        # Calculate silhouette score for cluster cohesion
        try:
            sil_score = __import__('sklearn.metrics').silhouette_score(embeddings, labels, metric='cosine')
        except:
            print("install scikit-learn pip install scikit-learn for better results")
            sil_score = 0

        # Calculate Davies-Bouldin score for cluster separation
        try:
            db_score = -__import__('sklearn.metrics').davies_bouldin_score(embeddings, labels)  # Negated as lower is better
        except:
            print("install scikit-learn pip install scikit-learn for better results")
            db_score = 0

        # Calculate query relevance if provided
        query_score = 0
        if query_embedding is not None:
            unique_labels = set(labels) - {-1}
            if unique_labels:
                query_sims = []
                for label in unique_labels:
                    cluster_mask = labels == label
                    cluster_embeddings = embeddings[cluster_mask]
                    cluster_centroid = np.mean(cluster_embeddings, axis=0)
                    query_sims.append(np.dot(cluster_centroid, query_embedding))
                query_score = np.mean(query_sims)

        # Combine scores with weights
        combined_score = (
            0.4 * sil_score +
            0.3 * db_score +
            0.3 * query_score
        )

        return combined_score

    @staticmethod
    def _fallback_clustering(
        chunks: list[Chunk],
        query_embedding: np.ndarray | None = None
    ) -> dict[int, list[Chunk]]:
        """
        Simple fallback clustering when HDBSCAN fails
        """
        if query_embedding is not None:
            # Sort by query relevance
            chunks_with_scores = [
                (chunk, np.dot(chunk.embedding, query_embedding))
                for chunk in chunks
            ]
            chunks_with_scores.sort(key=lambda x: x[1], reverse=True)
            chunks = [c for c, _ in chunks_with_scores]

        # Create fixed-size clusters
        clusters = {}
        cluster_size = max(2, len(chunks) // 5)

        for i in range(0, len(chunks), cluster_size):
            clusters[len(clusters)] = chunks[i:i + cluster_size]

        return clusters

    @staticmethod
    def _assign_noise_points(
        noise_chunks: list[Chunk],
        clusters: dict[int, list[Chunk]],
        query_embedding: np.ndarray | None = None
    ) -> None:
        """
        Assign noise points to nearest clusters
        """
        if not clusters:
            clusters[0] = noise_chunks
            return

        for chunk in noise_chunks:
            best_cluster = None
            best_similarity = float('-inf')

            for cluster_id, cluster_chunks in clusters.items():
                cluster_embeddings = np.vstack([c.embedding for c in cluster_chunks])
                cluster_centroid = np.mean(cluster_embeddings, axis=0)

                similarity = np.dot(chunk.embedding, cluster_centroid)

                # Consider query relevance in assignment if available
                if query_embedding is not None:
                    query_sim = np.dot(chunk.embedding, query_embedding)
                    similarity = 0.7 * similarity + 0.3 * query_sim

                if similarity > best_similarity:
                    best_similarity = similarity
                    best_cluster = cluster_id

            if best_cluster is not None:
                clusters[best_cluster].append(chunk)

    @staticmethod
    def _generate_topic_summary(
        chunks: list[Chunk],
        query_embedding: np.ndarray,
        max_sentences=3
    ) -> str:
        """Generate a summary for a topic using most representative chunks"""
        if not chunks:
            return ""

        # Find chunks most similar to cluster centroid
        embeddings = np.vstack([chunk.embedding for chunk in chunks])
        centroid = embeddings.mean(axis=0)

        # Calculate similarities to both centroid and query
        centroid_sims = np.dot(embeddings, centroid)
        query_sims = np.dot(embeddings, query_embedding)

        # Combine both similarities
        combined_sims = 0.7 * centroid_sims + 0.3 * query_sims

        # Select top sentences from most representative chunks
        top_indices = np.argsort(combined_sims)[-max_sentences:]
        summary_chunks = [chunks[i] for i in top_indices]

        # Extract key sentences
        sentences = []
        for chunk in summary_chunks:
            sentences.extend(sent.strip() for sent in chunk.text.split('.') if sent.strip())

        return '. '.join(sentences[:max_sentences]) + '.'

    async def retrieve_with_overview(
        self,
        query: str,
        query_embedding=None,
        k: int = 5,
        min_similarity: float = 0.2,
        max_sentences: int = 5,
        cross_ref_depth: int = 2,
        max_cross_refs: int = 10  # New parameter to control cross-reference count
    ) -> RetrievalResult:
        """Enhanced retrieval with better cross-reference handling"""
        # Get initial results with query embedding
        if query_embedding is None:
            query_embedding = (await self._get_embeddings([query]))[0]
        initial_results = await self.retrieve(query_embedding=query_embedding, k=k, min_similarity=min_similarity)

        if not initial_results:
            return RetrievalResult([], [], {})

        # Find cross-references with similarity scoring
        initial_ids = {self.vdb.chunks.index(chunk) for chunk in initial_results}
        related_ids = self._find_cross_references(
            initial_ids,
            depth=cross_ref_depth,
            query_embedding=query_embedding  # Pass query embedding for relevance scoring
        )

        # Get all relevant chunks with smarter filtering
        all_chunks = self.vdb.chunks
        all_relevant_chunks = initial_results + [
            chunk for i, chunk in enumerate(all_chunks)
            if i in related_ids and self._is_relevant_cross_ref(
                chunk,
                query_embedding,
                initial_results
            )
        ]

        # Enhanced clustering with dynamic cluster size
        clusters = self._cluster_chunks(
            all_relevant_chunks,
            query_embedding=query_embedding
        )

        # Fallback: If no clusters are found, treat all relevant chunks as a single cluster.
        if not clusters:
            print("No clusters found. Falling back to using all relevant chunks as a single cluster.")
            clusters = {0: all_relevant_chunks}

        # Generate summaries and organize results
        overview = []
        cross_references = {}

        for cluster_id, cluster_chunks in clusters.items():
            summary = self._generate_topic_summary(
                cluster_chunks,
                query_embedding,
                max_sentences=max_sentences  # Increased for more context
            )

            # Enhanced chunk sorting with combined scoring
            sorted_chunks = self._sort_chunks_by_relevance(
                cluster_chunks,
                query_embedding,
                initial_results
            )

            # Separate direct matches and cross-references
            direct_matches_ = [{'text':c.text, 'metadata':c.metadata} for c in sorted_chunks if c in initial_results]
            direct_matches = []
            for match in direct_matches_:
                if match in direct_matches:
                    continue
                direct_matches.append(match)
            cross_refs_ = [c for c in sorted_chunks if c not in initial_results]
            cross_refs = []
            for match in cross_refs_:
                if match in cross_refs:
                    continue
                cross_refs.append(match)
            # Limit cross-references while maintaining diversity
            selected_cross_refs = self._select_diverse_cross_refs(
                cross_refs,
                max_cross_refs,
                query_embedding
            )

            topic_info = {
                'topic_id': cluster_id,
                'summary': summary,
                'main_chunks': [x for x in direct_matches[:3]],
                'chunk_count': len(cluster_chunks),
                'relevance_score': self._calculate_topic_relevance(
                    cluster_chunks,
                    query_embedding
                )
            }
            overview.append(topic_info)

            if selected_cross_refs:
                cross_references[f"topic_{cluster_id}"] = selected_cross_refs

        # Sort overview by relevance score
        overview.sort(key=lambda x: x['relevance_score'], reverse=True)

        return RetrievalResult(
            overview=overview,
            details=initial_results,
            cross_references=cross_references
        )

    def _find_cross_references(
        self,
        chunk_ids: set[int],
        depth: int,
        query_embedding: np.ndarray
    ) -> set[int]:
        """Enhanced cross-reference finding with relevance scoring"""
        related_ids = set(chunk_ids)
        current_depth = 0
        frontier = set(chunk_ids)

        while current_depth < depth and frontier:
            new_frontier = set()
            for chunk_id in frontier:
                if chunk_id in self.similarity_graph:
                    # Score potential cross-references by relevance
                    candidates = self.similarity_graph[chunk_id] - related_ids
                    scored_candidates = [
                        (cid, self._calculate_topic_relevance(
                            [self.vdb.chunks[cid]],
                            query_embedding
                        ))
                        for cid in candidates
                    ]

                    # Filter by relevance threshold
                    relevant_candidates = {
                        cid for cid, score in scored_candidates
                        if score > 0.5  # Adjustable threshold
                    }
                    new_frontier.update(relevant_candidates)

            related_ids.update(new_frontier)
            frontier = new_frontier
            current_depth += 1

        return related_ids

    @staticmethod
    def _is_relevant_cross_ref(
        chunk: Chunk,
        query_embedding: np.ndarray,
        initial_results: list[Chunk]
    ) -> bool:
        """Determine if a cross-reference is relevant enough to include"""
        # Calculate similarity to query
        query_similarity = np.dot(chunk.embedding, query_embedding)

        # Calculate similarity to initial results
        initial_similarities = [
            np.dot(chunk.embedding, r.embedding) for r in initial_results
        ]
        max_initial_similarity = max(initial_similarities)

        # Combined relevance score
        relevance_score = 0.7 * query_similarity + 0.3 * max_initial_similarity

        return relevance_score > 0.6  # Adjustable threshold

    @staticmethod
    def _select_diverse_cross_refs(
        cross_refs: list[Chunk],
        max_count: int,
        query_embedding: np.ndarray
    ) -> list[Chunk]:
        """Select diverse and relevant cross-references"""
        if not cross_refs or len(cross_refs) <= max_count:
            return cross_refs

        # Calculate diversity scores
        embeddings = np.vstack([c.embedding for c in cross_refs])
        similarities = np.dot(embeddings, embeddings.T)

        selected = []
        remaining = list(enumerate(cross_refs))

        while len(selected) < max_count and remaining:
            # Score remaining chunks by relevance and diversity
            scores = []
            for idx, chunk in remaining:
                relevance = np.dot(chunk.embedding, query_embedding)
                diversity = 1.0
                if selected:
                    # Calculate diversity penalty based on similarity to selected chunks
                    selected_similarities = [
                        similarities[idx][list(cross_refs).index(s)]
                        for s in selected
                    ]
                    diversity = 1.0 - max(selected_similarities)

                combined_score = 0.7 * relevance + 0.3 * diversity
                scores.append((combined_score, idx, chunk))

            # Select the highest scoring chunk
            scores.sort(reverse=True)
            _, idx, chunk = scores[0]
            selected.append(chunk)
            remaining = [(i, c) for i, c in remaining if i != idx]

        return selected

    @staticmethod
    def _calculate_topic_relevance(
        chunks: list[Chunk],
        query_embedding: np.ndarray,
    ) -> float:
        """Calculate overall topic relevance score"""
        if not chunks:
            return 0.0

        similarities = [
            np.dot(chunk.embedding, query_embedding) for chunk in chunks
        ]
        return np.mean(similarities)

    @staticmethod
    def _sort_chunks_by_relevance(
        chunks: list[Chunk],
        query_embedding: np.ndarray,
        initial_results: list[Chunk]
    ) -> list[Chunk]:
        """Sort chunks by combined relevance score"""
        scored_chunks = []
        for chunk in chunks:
            query_similarity = np.dot(chunk.embedding, query_embedding)
            initial_similarities = [
                np.dot(chunk.embedding, r.embedding)
                for r in initial_results
            ]
            max_initial_similarity = max(initial_similarities) if initial_similarities else 0

            # Combined score favoring query relevance
            combined_score = 0.7 * query_similarity + 0.3 * max_initial_similarity
            scored_chunks.append((combined_score, chunk))

        scored_chunks.sort(reverse=True)
        return [chunk for _, chunk in scored_chunks]

    async def query_concepts(self, query: str) -> dict[str, any]:
        """Query concepts extracted from the knowledge base"""
        return await self.concept_extractor.query_concepts(query)

    async def unified_retrieve(
        self,
        query: str,
        k: int = 5,
        min_similarity: float = 0.2,
        cross_ref_depth: int = 2,
        max_cross_refs: int = 10,
        max_sentences: int = 10
    ) -> dict[str, Any]:
        """
        Unified retrieval function that combines concept querying, retrieval with overview,
        and basic retrieval, then generates a comprehensive summary using LLM.

        Args:
            query: Search query string
            k: Number of primary results to retrieve
            min_similarity: Minimum similarity threshold for retrieval
            cross_ref_depth: Depth for cross-reference search
            max_cross_refs: Maximum number of cross-references per topic
            max_sentences: Maximum number Sentences in the main summary text

        Returns:
            Dictionary containing comprehensive results including summary and details
        """
        # Get concept information
        concept_results = await self.concept_extractor.query_concepts(query)

        # Get retrieval overview

        query_embedding = (await self._get_embeddings([query]))[0]
        overview_results = await self.retrieve_with_overview(
            query=query,
            query_embedding=query_embedding,
            k=k,
            min_similarity=min_similarity,
            cross_ref_depth=cross_ref_depth,
            max_cross_refs=max_cross_refs,
            max_sentences=max_sentences
        )

        # Get basic retrieval results
        basic_results = await self.retrieve(
            query_embedding=query_embedding,
            k=k,
            min_similarity=min_similarity
        )
        if len(basic_results) == 0:
            return {}
        if len(basic_results) == 1 and isinstance(basic_results[0], str) and basic_results[0].endswith('[]\n - []\n - []'):
            return {}

        # Prepare context for LLM summary
        context = {
            "concepts": {
                "main_concepts": concept_results.get("concepts", {}),
                "relationships": concept_results.get("relationships", []),
                "concept_groups": concept_results.get("groups", [])
            },
            "topics": [
                {
                    "id": topic["topic_id"],
                    "summary": topic["summary"],
                    "relevance": topic["relevance_score"],
                    "chunk_count": topic["chunk_count"]
                }
                for topic in overview_results.overview
            ],
            "key_chunks": [
                {
                    "text": chunk.text,
                    "metadata": chunk.metadata
                }
                for chunk in basic_results
            ]
        }

        # Generate comprehensive summary using LLM
        system_prompt = """
        Analyze the provided search results and generate a comprehensive summary
        that includes:
        1. Main concepts and their relationships
        2. Key topics and their relevance
        3. Most important findings and insights
        4. Cross-references and connections between topics
        5. Potential gaps or areas for further investigation

        Format the response as a JSON object with these sections.
        """

        prompt = f"""
        Query: {query}

        Context:
        {json.dumps(context, indent=2)}

        Generate a comprehensive analysis and summary following the structure:
        """

        try:
            from toolboxv2.mods.isaa.extras.adapter import litellm_complete
            # await asyncio.sleep(0.25)
            llm_response = await litellm_complete(
                model_name=self.model_name,
                prompt=prompt,
                system_prompt=system_prompt,
                response_format=DataModel,
            )
            summary_analysis = json.loads(llm_response)
        except Exception as e:
            get_logger().error(f"Error generating summary: {str(e)}")
            summary_analysis = {
                "main_summary": "Error generating summary",
                "error": str(e)
            }

        # Compile final results
        return {
            "summary": summary_analysis,
            "raw_results": {
                "concepts": concept_results,
                "overview": {
                    "topics": overview_results.overview,
                    "cross_references": overview_results.cross_references
                },
                "relevant_chunks": [
                    {
                        "text": chunk.text,
                        "metadata": chunk.metadata,
                        "cluster_id": chunk.cluster_id
                    }
                    for chunk in basic_results
                ]
            },
            "metadata": {
                "query": query,
                "timestamp": time.time(),
                "retrieval_params": {
                    "k": k,
                    "min_similarity": min_similarity,
                    "cross_ref_depth": cross_ref_depth,
                    "max_cross_refs": max_cross_refs
                }
            }
        }

    def save(self, path: str) -> bytes | None:
        """
        Save the complete knowledge base to disk, including all sub-components

        Args:
            path (str): Path where the knowledge base will be saved
        """
        try:
            data = {
                # Core components
                'vdb': self.vdb.save(),
                'vis_kwargs': self.vis_kwargs,
                'vis_class': self.vis_class,
                'existing_hashes': self.existing_hashes,

                # Configuration parameters
                'embedding_dim': self.embedding_dim,
                'similarity_threshold': self.similarity_threshold,
                'batch_size': self.batch_size,
                'n_clusters': self.n_clusters,
                'deduplication_threshold': self.deduplication_threshold,
                'model_name': self.model_name,
                'embedding_model': self.embedding_model,

                # Cache and graph data
                'similarity_graph': self.similarity_graph,
                'sto': self.sto,

                # Text splitter configuration
                'text_splitter_config': {
                    'chunk_size': self.text_splitter.chunk_size,
                    'chunk_overlap': self.text_splitter.chunk_overlap,
                    'separator': self.text_splitter.separator
                },

                # Concept extractor data
                'concept_graph': {
                    'concepts': {
                        name: {
                            'name': concept.name,
                            'category': concept.category,
                            'relationships': {k: list(v) for k, v in concept.relationships.items()},
                            'importance_score': concept.importance_score,
                            'context_snippets': concept.context_snippets,
                            'metadata': concept.metadata
                        }
                        for name, concept in self.concept_extractor.concept_graph.concepts.items()
                    }
                }
            }
            b = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)

            if path is None:
                return b

            path = Path(path)
            tmp = path.with_suffix(path.suffix + ".tmp") if path.suffix else path.with_name(path.name + ".tmp")

            try:
                # Schreibe zuerst in eine temporäre Datei
                with open(tmp, "wb") as f:
                    f.write(b)
                    f.flush()
                    os.fsync(f.fileno())  # sicherstellen, dass die Daten auf Platte sind
                # Atomischer Austausch
                os.replace(tmp, path)
            finally:
                # Aufräumen falls tmp noch existiert (bei Fehlern)
                if tmp.exists():
                    with contextlib.suppress(Exception):
                        tmp.unlink()
            return None
            # print(f"Knowledge base successfully saved to {path} with {len(self.concept_extractor.concept_graph.concepts.items())} concepts")

        except Exception as e:
            print(f"Error saving knowledge base: {str(e)}")
            raise
    def init_vdb(self, db:AbstractVectorStore=AbstractVectorStore):
        pass
    @classmethod
    def load(cls, path: str | bytes) -> 'KnowledgeBase':
        """
        Load a complete knowledge base from disk, including all sub-components

        Args:
            path (str): Path from where to load the knowledge base

        Returns:
            KnowledgeBase: A fully restored knowledge base instance
        """
        try:
            if isinstance(path, bytes | bytearray | memoryview):
                data_bytes = bytes(path)
                try:
                    data = pickle.loads(data_bytes)
                except Exception as e:
                    raise EOFError(f"Fehler beim pickle.loads von bytes: {e}") from e
            else:
                p = Path(path)
                if not p.exists():
                    raise FileNotFoundError(f"{p} existiert nicht")
                size = p.stat().st_size
                if size == 0:
                    raise EOFError(f"{p} ist leer (0 bytes)")
                try:
                    with open(p, "rb") as f:
                        try:
                            data = pickle.load(f)
                        except EOFError as e:
                            # Debug info: erste bytes ausgeben
                            f.seek(0)
                            snippet = f.read(128)
                            raise EOFError(
                                f"EOFError beim Laden {p} (Größe {size} bytes). Erste 128 bytes: {snippet!r}") from e

                except Exception as e:
                    raise ValueError(f"Invalid path type {e}") from e
            # Create new knowledge base instance with saved configuration
            kb = cls(
                embedding_dim=data['embedding_dim'],
                similarity_threshold=data['similarity_threshold'],
                batch_size=data['batch_size'],
                n_clusters=data['n_clusters'],
                deduplication_threshold=data['deduplication_threshold'],
                model_name=data['model_name'],
                embedding_model=data['embedding_model']
            )

            # Restore core components
            kb.init_vis(data.get('vis_class'), data.get('vis_kwargs'))
            kb.existing_hashes = data['existing_hashes']

            # Restore cache and graph data
            kb.similarity_graph = data.get('similarity_graph', {})
            kb.sto = data.get('sto', [])

            # Restore text splitter configuration
            splitter_config = data.get('text_splitter_config', {})
            kb.text_splitter = TextSplitter(
                chunk_size=splitter_config.get('chunk_size', 12_000),
                chunk_overlap=splitter_config.get('chunk_overlap', 200),
                separator=splitter_config.get('separator', '\n')
            )

            # Restore concept graph
            concept_data = data.get('concept_graph', {}).get('concepts', {})
            for concept_info in concept_data.values():
                concept = Concept(
                    name=concept_info['name'],
                    category=concept_info['category'],
                    relationships={k: set(v) for k, v in concept_info['relationships'].items()},
                    importance_score=concept_info['importance_score'],
                    context_snippets=concept_info['context_snippets'],
                    metadata=concept_info['metadata']
                )
                kb.concept_extractor.concept_graph.add_concept(concept)

            # print(f"Knowledge base successfully loaded from {path} with {len(concept_data)} concepts")
            return kb

        except Exception:
            print(f"Error loading knowledge base: {str(e)}")
            import traceback
            traceback.print_exception(e)
            raise

    async def vis(self,output_file: str = "concept_graph.html", get_output_html=False, get_output_net=False):

        if not self.concept_extractor.concept_graph.concepts:

            if len(self.sto) > 2:
                await self.add_data([t for (t, m) in self.sto], [m for (t, m) in self.sto], direct=True)
                # self.sto = []
            if not self.concept_extractor.concept_graph.concepts:
                print("NO Concepts defined and no data in sto")
                return None


        net = self.concept_extractor.concept_graph.convert_to_networkx()
        if get_output_net:
            return net
        return GraphVisualizer.visualize(net, output_file=output_file, get_output=get_output_html)
__init__(embedding_dim=256, similarity_threshold=0.61, batch_size=12, n_clusters=4, deduplication_threshold=0.85, model_name=os.getenv('SUMMARYMODEL'), embedding_model=os.getenv('DEFAULTMODELEMBEDDING'), vis_class='FaissVectorStore', vis_kwargs=None, requests_per_second=85.0, chunk_size=3600, chunk_overlap=130, separator='\n')

Initialize the knowledge base with given parameters

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
def __init__(self, embedding_dim: int = 256, similarity_threshold: float = 0.61, batch_size: int = 12,
             n_clusters: int = 4, deduplication_threshold: float = 0.85, model_name=os.getenv("SUMMARYMODEL"),
             embedding_model=os.getenv("DEFAULTMODELEMBEDDING"),
             vis_class:str | None = "FaissVectorStore",
             vis_kwargs:dict[str, Any] | None=None,
             requests_per_second=85.,
             chunk_size: int = 3600,
             chunk_overlap: int = 130,
             separator: str = "\n"
             ):
    """Initialize the knowledge base with given parameters"""

    self.existing_hashes: set[str] = set()
    self.embedding_model = embedding_model
    self.embedding_dim = embedding_dim
    self.similarity_threshold = similarity_threshold
    self.deduplication_threshold = deduplication_threshold
    if model_name == "openrouter/mistralai/mistral-nemo":
        batch_size = 9
        requests_per_second = 1.5
    self.batch_size = batch_size
    self.n_clusters = n_clusters
    self.model_name = model_name
    self.sto: list = []

    self.text_splitter = TextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap, separator=separator)
    self.similarity_graph = {}
    self.concept_extractor = ConceptExtractor(self, requests_per_second)

    self.vis_class = None
    self.vis_kwargs = None
    self.vdb = None
    self.init_vis(vis_class, vis_kwargs)
add_data(texts, metadata=None, direct=False) async

Enhanced version with smart splitting and clustering

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
async def add_data(
    self,
    texts: list[str],
    metadata: list[dict[str, Any]] | None = None, direct:bool = False
) -> tuple[int, int]:
    """Enhanced version with smart splitting and clustering"""
    if isinstance(texts, str):
        texts = [texts]
    if metadata is None:
        metadata = [{}] * len(texts)
    if isinstance(metadata, dict):
        metadata = [metadata]
    if len(texts) != len(metadata):
        raise ValueError("Length of texts and metadata must match")

    if not direct and len(texts) == 1 and len(texts[0]) < 10_000:
        if len(self.sto) < self.batch_size and len(texts) == 1:
            self.sto.append((texts[0], metadata[0]))
            return -1, -1
        if len(self.sto) >= self.batch_size:
            _ = [texts.append(t) or metadata.append([m]) for (t, m) in self.sto]
            self.sto = []

    # Split large texts
    split_texts = []
    split_metadata = []

    while Spinner("Saving Data to Memory", symbols='t'):

        for idx, text in enumerate(texts):
            chunks = self.text_splitter.split_text(text)
            split_texts.extend(chunks)

            # Adjust metadata for splits
            meta = metadata[idx] if metadata else {}
            if isinstance(meta, list):
                meta = meta[0]
            for i, _chunk in enumerate(chunks):
                chunk_meta = meta.copy()
                chunk_meta.update({
                    'chunk_index': i,
                    'total_chunks': len(chunks),
                    'original_text_id': idx
                })
                split_metadata.append(chunk_meta)

        return await self._add_data(split_texts, split_metadata)
compute_hash(text) staticmethod

Compute SHA-256 hash of text

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
681
682
683
684
@staticmethod
def compute_hash(text: str) -> str:
    """Compute SHA-256 hash of text"""
    return hashlib.sha256(text.encode('utf-8', errors='ignore')).hexdigest()
forget_irrelevant(irrelevant_concepts, similarity_threshold=None) async

Remove chunks similar to irrelevant concepts Returns: Number of chunks removed

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
async def forget_irrelevant(self, irrelevant_concepts: list[str], similarity_threshold: float | None=None) -> int:
    """
    Remove chunks similar to irrelevant concepts
    Returns: Number of chunks removed
    """
    if not irrelevant_concepts:
        return 0

    if similarity_threshold is None:
        similarity_threshold = self.similarity_threshold

    try:
        irrelevant_embeddings = await self._get_embeddings(irrelevant_concepts)
        initial_count = len(self.vdb.chunks)

        def is_relevant(chunk: Chunk) -> bool:
            similarities = np.dot(chunk.embedding, irrelevant_embeddings.T)
            do_keep = np.max(similarities) < similarity_threshold
            if do_keep:
                return True
            for c in chunk.metadata.get("concepts", []):
                if c in self.concept_extractor.concept_graph.concepts:
                    del self.concept_extractor.concept_graph.concepts[c]
            return False

        relevant_chunks = [chunk for chunk in self.vdb.chunks if is_relevant(chunk)]
        self.vdb.chunks = relevant_chunks
        self.existing_hashes = {chunk.content_hash for chunk in self.vdb.chunks}
        self.vdb.rebuild_index()

        return initial_count - len(self.vdb.chunks)

    except Exception as e:
        get_logger().error(f"Error forgetting irrelevant concepts: {str(e)}")
        raise
load(path) classmethod

Load a complete knowledge base from disk, including all sub-components

Parameters:

Name Type Description Default
path str

Path from where to load the knowledge base

required

Returns:

Name Type Description
KnowledgeBase KnowledgeBase

A fully restored knowledge base instance

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
@classmethod
def load(cls, path: str | bytes) -> 'KnowledgeBase':
    """
    Load a complete knowledge base from disk, including all sub-components

    Args:
        path (str): Path from where to load the knowledge base

    Returns:
        KnowledgeBase: A fully restored knowledge base instance
    """
    try:
        if isinstance(path, bytes | bytearray | memoryview):
            data_bytes = bytes(path)
            try:
                data = pickle.loads(data_bytes)
            except Exception as e:
                raise EOFError(f"Fehler beim pickle.loads von bytes: {e}") from e
        else:
            p = Path(path)
            if not p.exists():
                raise FileNotFoundError(f"{p} existiert nicht")
            size = p.stat().st_size
            if size == 0:
                raise EOFError(f"{p} ist leer (0 bytes)")
            try:
                with open(p, "rb") as f:
                    try:
                        data = pickle.load(f)
                    except EOFError as e:
                        # Debug info: erste bytes ausgeben
                        f.seek(0)
                        snippet = f.read(128)
                        raise EOFError(
                            f"EOFError beim Laden {p} (Größe {size} bytes). Erste 128 bytes: {snippet!r}") from e

            except Exception as e:
                raise ValueError(f"Invalid path type {e}") from e
        # Create new knowledge base instance with saved configuration
        kb = cls(
            embedding_dim=data['embedding_dim'],
            similarity_threshold=data['similarity_threshold'],
            batch_size=data['batch_size'],
            n_clusters=data['n_clusters'],
            deduplication_threshold=data['deduplication_threshold'],
            model_name=data['model_name'],
            embedding_model=data['embedding_model']
        )

        # Restore core components
        kb.init_vis(data.get('vis_class'), data.get('vis_kwargs'))
        kb.existing_hashes = data['existing_hashes']

        # Restore cache and graph data
        kb.similarity_graph = data.get('similarity_graph', {})
        kb.sto = data.get('sto', [])

        # Restore text splitter configuration
        splitter_config = data.get('text_splitter_config', {})
        kb.text_splitter = TextSplitter(
            chunk_size=splitter_config.get('chunk_size', 12_000),
            chunk_overlap=splitter_config.get('chunk_overlap', 200),
            separator=splitter_config.get('separator', '\n')
        )

        # Restore concept graph
        concept_data = data.get('concept_graph', {}).get('concepts', {})
        for concept_info in concept_data.values():
            concept = Concept(
                name=concept_info['name'],
                category=concept_info['category'],
                relationships={k: set(v) for k, v in concept_info['relationships'].items()},
                importance_score=concept_info['importance_score'],
                context_snippets=concept_info['context_snippets'],
                metadata=concept_info['metadata']
            )
            kb.concept_extractor.concept_graph.add_concept(concept)

        # print(f"Knowledge base successfully loaded from {path} with {len(concept_data)} concepts")
        return kb

    except Exception:
        print(f"Error loading knowledge base: {str(e)}")
        import traceback
        traceback.print_exception(e)
        raise
query_concepts(query) async

Query concepts extracted from the knowledge base

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
1488
1489
1490
async def query_concepts(self, query: str) -> dict[str, any]:
    """Query concepts extracted from the knowledge base"""
    return await self.concept_extractor.query_concepts(query)
retrieve(query='', query_embedding=None, k=5, min_similarity=0.2, include_connected=True) async

Enhanced retrieval with connected information

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
async def retrieve(
    self,
    query: str="",
    query_embedding: np.ndarray | None = None,
    k: int = 5,
    min_similarity: float = 0.2,
    include_connected: bool = True
) -> list[Chunk]:
    """Enhanced retrieval with connected information"""
    if query_embedding is None:
        query_embedding = (await self._get_embeddings([query]))[0]
    k = min(k, len(self.vdb.chunks))
    if k <= 0:
        return []
    initial_results = self.vdb.search(query_embedding, k, min_similarity)

    if not include_connected or not initial_results:
        return initial_results

    # Find connected chunks
    connected_chunks = set()
    for chunk in initial_results:
        chunk_id = self.vdb.chunks.index(chunk)
        if chunk_id in self.similarity_graph:
            connected_chunks.update(self.similarity_graph[chunk_id])

    # Add connected chunks to results
    all_chunks = self.vdb.chunks
    additional_results = [all_chunks[i] for i in connected_chunks
                          if all_chunks[i] not in initial_results]

    # Sort by similarity to query
    all_results = initial_results + additional_results

    return sorted(
        all_results,
        key=lambda x: np.dot(x.embedding, query_embedding),
        reverse=True
    )[:k * 2]  # Return more results when including connected information
retrieve_with_overview(query, query_embedding=None, k=5, min_similarity=0.2, max_sentences=5, cross_ref_depth=2, max_cross_refs=10) async

Enhanced retrieval with better cross-reference handling

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
async def retrieve_with_overview(
    self,
    query: str,
    query_embedding=None,
    k: int = 5,
    min_similarity: float = 0.2,
    max_sentences: int = 5,
    cross_ref_depth: int = 2,
    max_cross_refs: int = 10  # New parameter to control cross-reference count
) -> RetrievalResult:
    """Enhanced retrieval with better cross-reference handling"""
    # Get initial results with query embedding
    if query_embedding is None:
        query_embedding = (await self._get_embeddings([query]))[0]
    initial_results = await self.retrieve(query_embedding=query_embedding, k=k, min_similarity=min_similarity)

    if not initial_results:
        return RetrievalResult([], [], {})

    # Find cross-references with similarity scoring
    initial_ids = {self.vdb.chunks.index(chunk) for chunk in initial_results}
    related_ids = self._find_cross_references(
        initial_ids,
        depth=cross_ref_depth,
        query_embedding=query_embedding  # Pass query embedding for relevance scoring
    )

    # Get all relevant chunks with smarter filtering
    all_chunks = self.vdb.chunks
    all_relevant_chunks = initial_results + [
        chunk for i, chunk in enumerate(all_chunks)
        if i in related_ids and self._is_relevant_cross_ref(
            chunk,
            query_embedding,
            initial_results
        )
    ]

    # Enhanced clustering with dynamic cluster size
    clusters = self._cluster_chunks(
        all_relevant_chunks,
        query_embedding=query_embedding
    )

    # Fallback: If no clusters are found, treat all relevant chunks as a single cluster.
    if not clusters:
        print("No clusters found. Falling back to using all relevant chunks as a single cluster.")
        clusters = {0: all_relevant_chunks}

    # Generate summaries and organize results
    overview = []
    cross_references = {}

    for cluster_id, cluster_chunks in clusters.items():
        summary = self._generate_topic_summary(
            cluster_chunks,
            query_embedding,
            max_sentences=max_sentences  # Increased for more context
        )

        # Enhanced chunk sorting with combined scoring
        sorted_chunks = self._sort_chunks_by_relevance(
            cluster_chunks,
            query_embedding,
            initial_results
        )

        # Separate direct matches and cross-references
        direct_matches_ = [{'text':c.text, 'metadata':c.metadata} for c in sorted_chunks if c in initial_results]
        direct_matches = []
        for match in direct_matches_:
            if match in direct_matches:
                continue
            direct_matches.append(match)
        cross_refs_ = [c for c in sorted_chunks if c not in initial_results]
        cross_refs = []
        for match in cross_refs_:
            if match in cross_refs:
                continue
            cross_refs.append(match)
        # Limit cross-references while maintaining diversity
        selected_cross_refs = self._select_diverse_cross_refs(
            cross_refs,
            max_cross_refs,
            query_embedding
        )

        topic_info = {
            'topic_id': cluster_id,
            'summary': summary,
            'main_chunks': [x for x in direct_matches[:3]],
            'chunk_count': len(cluster_chunks),
            'relevance_score': self._calculate_topic_relevance(
                cluster_chunks,
                query_embedding
            )
        }
        overview.append(topic_info)

        if selected_cross_refs:
            cross_references[f"topic_{cluster_id}"] = selected_cross_refs

    # Sort overview by relevance score
    overview.sort(key=lambda x: x['relevance_score'], reverse=True)

    return RetrievalResult(
        overview=overview,
        details=initial_results,
        cross_references=cross_references
    )
save(path)

Save the complete knowledge base to disk, including all sub-components

Parameters:

Name Type Description Default
path str

Path where the knowledge base will be saved

required
Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
def save(self, path: str) -> bytes | None:
    """
    Save the complete knowledge base to disk, including all sub-components

    Args:
        path (str): Path where the knowledge base will be saved
    """
    try:
        data = {
            # Core components
            'vdb': self.vdb.save(),
            'vis_kwargs': self.vis_kwargs,
            'vis_class': self.vis_class,
            'existing_hashes': self.existing_hashes,

            # Configuration parameters
            'embedding_dim': self.embedding_dim,
            'similarity_threshold': self.similarity_threshold,
            'batch_size': self.batch_size,
            'n_clusters': self.n_clusters,
            'deduplication_threshold': self.deduplication_threshold,
            'model_name': self.model_name,
            'embedding_model': self.embedding_model,

            # Cache and graph data
            'similarity_graph': self.similarity_graph,
            'sto': self.sto,

            # Text splitter configuration
            'text_splitter_config': {
                'chunk_size': self.text_splitter.chunk_size,
                'chunk_overlap': self.text_splitter.chunk_overlap,
                'separator': self.text_splitter.separator
            },

            # Concept extractor data
            'concept_graph': {
                'concepts': {
                    name: {
                        'name': concept.name,
                        'category': concept.category,
                        'relationships': {k: list(v) for k, v in concept.relationships.items()},
                        'importance_score': concept.importance_score,
                        'context_snippets': concept.context_snippets,
                        'metadata': concept.metadata
                    }
                    for name, concept in self.concept_extractor.concept_graph.concepts.items()
                }
            }
        }
        b = pickle.dumps(data, protocol=pickle.HIGHEST_PROTOCOL)

        if path is None:
            return b

        path = Path(path)
        tmp = path.with_suffix(path.suffix + ".tmp") if path.suffix else path.with_name(path.name + ".tmp")

        try:
            # Schreibe zuerst in eine temporäre Datei
            with open(tmp, "wb") as f:
                f.write(b)
                f.flush()
                os.fsync(f.fileno())  # sicherstellen, dass die Daten auf Platte sind
            # Atomischer Austausch
            os.replace(tmp, path)
        finally:
            # Aufräumen falls tmp noch existiert (bei Fehlern)
            if tmp.exists():
                with contextlib.suppress(Exception):
                    tmp.unlink()
        return None
        # print(f"Knowledge base successfully saved to {path} with {len(self.concept_extractor.concept_graph.concepts.items())} concepts")

    except Exception as e:
        print(f"Error saving knowledge base: {str(e)}")
        raise
unified_retrieve(query, k=5, min_similarity=0.2, cross_ref_depth=2, max_cross_refs=10, max_sentences=10) async

Unified retrieval function that combines concept querying, retrieval with overview, and basic retrieval, then generates a comprehensive summary using LLM.

Parameters:

Name Type Description Default
query str

Search query string

required
k int

Number of primary results to retrieve

5
min_similarity float

Minimum similarity threshold for retrieval

0.2
cross_ref_depth int

Depth for cross-reference search

2
max_cross_refs int

Maximum number of cross-references per topic

10
max_sentences int

Maximum number Sentences in the main summary text

10

Returns:

Type Description
dict[str, Any]

Dictionary containing comprehensive results including summary and details

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
async def unified_retrieve(
    self,
    query: str,
    k: int = 5,
    min_similarity: float = 0.2,
    cross_ref_depth: int = 2,
    max_cross_refs: int = 10,
    max_sentences: int = 10
) -> dict[str, Any]:
    """
    Unified retrieval function that combines concept querying, retrieval with overview,
    and basic retrieval, then generates a comprehensive summary using LLM.

    Args:
        query: Search query string
        k: Number of primary results to retrieve
        min_similarity: Minimum similarity threshold for retrieval
        cross_ref_depth: Depth for cross-reference search
        max_cross_refs: Maximum number of cross-references per topic
        max_sentences: Maximum number Sentences in the main summary text

    Returns:
        Dictionary containing comprehensive results including summary and details
    """
    # Get concept information
    concept_results = await self.concept_extractor.query_concepts(query)

    # Get retrieval overview

    query_embedding = (await self._get_embeddings([query]))[0]
    overview_results = await self.retrieve_with_overview(
        query=query,
        query_embedding=query_embedding,
        k=k,
        min_similarity=min_similarity,
        cross_ref_depth=cross_ref_depth,
        max_cross_refs=max_cross_refs,
        max_sentences=max_sentences
    )

    # Get basic retrieval results
    basic_results = await self.retrieve(
        query_embedding=query_embedding,
        k=k,
        min_similarity=min_similarity
    )
    if len(basic_results) == 0:
        return {}
    if len(basic_results) == 1 and isinstance(basic_results[0], str) and basic_results[0].endswith('[]\n - []\n - []'):
        return {}

    # Prepare context for LLM summary
    context = {
        "concepts": {
            "main_concepts": concept_results.get("concepts", {}),
            "relationships": concept_results.get("relationships", []),
            "concept_groups": concept_results.get("groups", [])
        },
        "topics": [
            {
                "id": topic["topic_id"],
                "summary": topic["summary"],
                "relevance": topic["relevance_score"],
                "chunk_count": topic["chunk_count"]
            }
            for topic in overview_results.overview
        ],
        "key_chunks": [
            {
                "text": chunk.text,
                "metadata": chunk.metadata
            }
            for chunk in basic_results
        ]
    }

    # Generate comprehensive summary using LLM
    system_prompt = """
    Analyze the provided search results and generate a comprehensive summary
    that includes:
    1. Main concepts and their relationships
    2. Key topics and their relevance
    3. Most important findings and insights
    4. Cross-references and connections between topics
    5. Potential gaps or areas for further investigation

    Format the response as a JSON object with these sections.
    """

    prompt = f"""
    Query: {query}

    Context:
    {json.dumps(context, indent=2)}

    Generate a comprehensive analysis and summary following the structure:
    """

    try:
        from toolboxv2.mods.isaa.extras.adapter import litellm_complete
        # await asyncio.sleep(0.25)
        llm_response = await litellm_complete(
            model_name=self.model_name,
            prompt=prompt,
            system_prompt=system_prompt,
            response_format=DataModel,
        )
        summary_analysis = json.loads(llm_response)
    except Exception as e:
        get_logger().error(f"Error generating summary: {str(e)}")
        summary_analysis = {
            "main_summary": "Error generating summary",
            "error": str(e)
        }

    # Compile final results
    return {
        "summary": summary_analysis,
        "raw_results": {
            "concepts": concept_results,
            "overview": {
                "topics": overview_results.overview,
                "cross_references": overview_results.cross_references
            },
            "relevant_chunks": [
                {
                    "text": chunk.text,
                    "metadata": chunk.metadata,
                    "cluster_id": chunk.cluster_id
                }
                for chunk in basic_results
            ]
        },
        "metadata": {
            "query": query,
            "timestamp": time.time(),
            "retrieval_params": {
                "k": k,
                "min_similarity": min_similarity,
                "cross_ref_depth": cross_ref_depth,
                "max_cross_refs": max_cross_refs
            }
        }
    }
RelevanceAssessment

Bases: BaseModel

Represents an assessment of the relevance of the data in relation to a specific query.

Attributes:

Name Type Description
query_alignment float

A float representing the alignment between the query and the data.

confidence_score float

A float indicating the confidence level in the alignment.

coverage_analysis str

A textual description analyzing the data coverage.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
139
140
141
142
143
144
145
146
147
148
149
150
class RelevanceAssessment(BaseModel):
    """
    Represents an assessment of the relevance of the data in relation to a specific query.

    Attributes:
        query_alignment (float): A float representing the alignment between the query and the data.
        confidence_score (float): A float indicating the confidence level in the alignment.
        coverage_analysis (str): A textual description analyzing the data coverage.
    """
    query_alignment: float
    confidence_score: float
    coverage_analysis: str
RetrievalResult dataclass

Structure for organizing retrieval results

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
37
38
39
40
41
42
@dataclass
class RetrievalResult:
    """Structure for organizing retrieval results"""
    overview: list[dict[str, any]]  # List of topic summaries
    details: list[Chunk]  # Detailed chunks
    cross_references: dict[str, list[Chunk]]  # Related chunks by topic
TConcept

Bases: BaseModel

Represents the criteria or target parameters for concept selection and filtering.

Attributes:

Name Type Description
min_importance float

The minimum importance score a concept must have to be considered.

target_concepts List[str]

A list of names of target concepts to focus on.

relationship_types List[str]

A list of relationship types to be considered in the analysis.

categories List[str]

A list of concept categories to filter or group the concepts.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
86
87
88
89
90
91
92
93
94
95
96
97
98
99
class TConcept(BaseModel):
    """
    Represents the criteria or target parameters for concept selection and filtering.

    Attributes:
        min_importance (float): The minimum importance score a concept must have to be considered.
        target_concepts (List[str]): A list of names of target concepts to focus on.
        relationship_types (List[str]): A list of relationship types to be considered in the analysis.
        categories (List[str]): A list of concept categories to filter or group the concepts.
    """
    min_importance: float
    target_concepts: list[str]
    relationship_types: list[str]
    categories: list[str]
TextSplitter
Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
class TextSplitter:
    def __init__(
        self,
        chunk_size: int = 3600,
        chunk_overlap: int = 130,
        separator: str = "\n"
    ):
        self.chunk_size = chunk_size
        self.chunk_overlap = chunk_overlap
        self.separator = separator

    def approximate(self, text_len: int) -> float:
        """
        Approximate the number of chunks and average chunk size for a given text length

        Args:
            text_len (int): Length of the text to be split

        Returns:
            Tuple[int, int]: (number_of_chunks, approximate_chunk_size)
        """
        if text_len <= self.chunk_size:
            return 1, text_len

        # Handle extreme overlap cases
        if self.chunk_overlap >= self.chunk_size:
            estimated_chunks = text_len
            return estimated_chunks, 1

        # Calculate based on overlap ratio
        overlap_ratio = self.chunk_overlap / self.chunk_size
        base_chunks = text_len / self.chunk_size
        estimated_chunks = base_chunks * 2 / (overlap_ratio if overlap_ratio > 0 else 1)

        # print('#',estimated_chunks, base_chunks, overlap_ratio)
        # Calculate average chunk size
        avg_chunk_size = max(1, text_len / estimated_chunks)

        return estimated_chunks * avg_chunk_size

    def split_text(self, text: str) -> list[str]:
        """Split text into chunks with overlap"""
        # Clean and normalize text
        text = re.sub(r'\s+', ' ', text).strip()

        # If text is shorter than chunk_size, return as is
        if len(text) <= self.chunk_size:
            return [text]

        chunks = []
        start = 0

        while start < len(text):
            # Find end of chunk
            end = start + self.chunk_size

            if end >= len(text):
                chunks.append(text[start:])
                break

            # Try to find a natural break point
            last_separator = text.rfind(self.separator, start, end)
            if last_separator != -1:
                end = last_separator

            # Add chunk
            chunks.append(text[start:end])

            # Calculate allowed overlap for this chunk
            chunk_length = end - start
            allowed_overlap = min(self.chunk_overlap, chunk_length - 1)

            # Move start position considering adjusted overlap
            start = end - allowed_overlap

        return chunks
approximate(text_len)

Approximate the number of chunks and average chunk size for a given text length

Parameters:

Name Type Description Default
text_len int

Length of the text to be split

required

Returns:

Type Description
float

Tuple[int, int]: (number_of_chunks, approximate_chunk_size)

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
def approximate(self, text_len: int) -> float:
    """
    Approximate the number of chunks and average chunk size for a given text length

    Args:
        text_len (int): Length of the text to be split

    Returns:
        Tuple[int, int]: (number_of_chunks, approximate_chunk_size)
    """
    if text_len <= self.chunk_size:
        return 1, text_len

    # Handle extreme overlap cases
    if self.chunk_overlap >= self.chunk_size:
        estimated_chunks = text_len
        return estimated_chunks, 1

    # Calculate based on overlap ratio
    overlap_ratio = self.chunk_overlap / self.chunk_size
    base_chunks = text_len / self.chunk_size
    estimated_chunks = base_chunks * 2 / (overlap_ratio if overlap_ratio > 0 else 1)

    # print('#',estimated_chunks, base_chunks, overlap_ratio)
    # Calculate average chunk size
    avg_chunk_size = max(1, text_len / estimated_chunks)

    return estimated_chunks * avg_chunk_size
split_text(text)

Split text into chunks with overlap

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
def split_text(self, text: str) -> list[str]:
    """Split text into chunks with overlap"""
    # Clean and normalize text
    text = re.sub(r'\s+', ' ', text).strip()

    # If text is shorter than chunk_size, return as is
    if len(text) <= self.chunk_size:
        return [text]

    chunks = []
    start = 0

    while start < len(text):
        # Find end of chunk
        end = start + self.chunk_size

        if end >= len(text):
            chunks.append(text[start:])
            break

        # Try to find a natural break point
        last_separator = text.rfind(self.separator, start, end)
        if last_separator != -1:
            end = last_separator

        # Add chunk
        chunks.append(text[start:end])

        # Calculate allowed overlap for this chunk
        chunk_length = end - start
        allowed_overlap = min(self.chunk_overlap, chunk_length - 1)

        # Move start position considering adjusted overlap
        start = end - allowed_overlap

    return chunks
TopicInsights

Bases: BaseModel

Represents insights related to various topics.

Attributes:

Name Type Description
primary_topics list[str]

A list of main topics addressed.

cross_references list[str]

A list of cross-references that connect different topics.

knowledge_gaps list[str]

A list of identified gaps in the current knowledge.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
125
126
127
128
129
130
131
132
133
134
135
136
class TopicInsights(BaseModel):
    """
    Represents insights related to various topics.

    Attributes:
        primary_topics (list[str]): A list of main topics addressed.
        cross_references (list[str]): A list of cross-references that connect different topics.
        knowledge_gaps (list[str]): A list of identified gaps in the current knowledge.
    """
    primary_topics: list[str]
    cross_references: list[str]
    knowledge_gaps: list[str]
rConcept

Bases: BaseModel

Represents a key concept with its relationships and associated metadata.

Attributes:

Name Type Description
name str

The name of the concept.

category str

The category of the concept (e.g., 'technical', 'domain', 'method', etc.).

relationships Dict[str, List[str]]

A mapping where each key is a type of relationship and the value is a list of related concept names.

importance_score float

A numerical score representing the importance or relevance of the concept.

context_snippets List[str]

A list of text snippets providing context where the concept appears.

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
class rConcept(BaseModel):
    """
    Represents a key concept with its relationships and associated metadata.

    Attributes:
        name (str): The name of the concept.
        category (str): The category of the concept (e.g., 'technical', 'domain', 'method', etc.).
        relationships (Dict[str, List[str]]): A mapping where each key is a type of relationship and the
            value is a list of related concept names.
        importance_score (float): A numerical score representing the importance or relevance of the concept.
        context_snippets (List[str]): A list of text snippets providing context where the concept appears.
    """
    name: str
    category: str
    relationships: dict[str, list[str]]
    importance_score: float
    context_snippets: list[str]
normalize_vectors(vectors)

Normalize vectors to unit length

Source code in toolboxv2/mods/isaa/base/KnowledgeBase.py
52
53
54
55
def normalize_vectors(vectors: np.ndarray) -> np.ndarray:
    """Normalize vectors to unit length"""
    norms = np.linalg.norm(vectors, axis=1, keepdims=True)
    return np.divide(vectors, norms, where=norms != 0)
VectorStores

Vector store implementations for the toolboxv2 system.

taichiNumpyNumbaVectorStores
NumpyVectorStore

Bases: AbstractVectorStore

Source code in toolboxv2/mods/isaa/base/VectorStores/taichiNumpyNumbaVectorStores.py
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
class NumpyVectorStore(AbstractVectorStore):
    def __init__(self, use_gpu=False):
        self.embeddings = np.empty((0, 0))
        self.chunks = []
        # Initialize Taich


        self.normalized_embeddings = None

    def add_embeddings(self, embeddings: np.ndarray, chunks: list[Chunk]) -> None:
        if len(embeddings.shape) != 2:
            raise ValueError("Embeddings must be 2D array")
        if len(chunks) != embeddings.shape[0]:
            raise ValueError("Mismatch between embeddings and chunks count")

        if self.embeddings.size == 0:
            self.embeddings = embeddings
        else:
            if embeddings.shape[1] != self.embeddings.shape[1]:
                raise ValueError("Embedding dimensions must match")
            self.embeddings = np.vstack([self.embeddings, embeddings])
        self.chunks.extend(chunks)
        # Reset normalized embeddings cache
        self.normalized_embeddings = None

    def search(self, query_embedding: np.ndarray, k: int = 5, min_similarity: float = 0.7) -> list[Chunk]:
        if self.embeddings.size == 0:
            return []

        # Pre-compute normalized embeddings if not cached
        if self.normalized_embeddings is None:
            self._precompute_normalized_embeddings()

        # Normalize query
        query_norm = self._normalize_vector(query_embedding)

        # Enhanced Taichi kernel for similarity computation
        n = len(self.chunks)
        similarities = np.zeros(n, dtype=np.float32)

        @ti.kernel
        def compute_similarities_optimized(
            query: ti.types.ndarray(dtype=ti.f32),
            embeddings: ti.types.ndarray(dtype=ti.f32),
            similarities: ti.types.ndarray(dtype=ti.f32),
            n: ti.i32,
            dim: ti.i32
        ):
            ti.loop_config(block_dim=256)
            for i in range(n):
                dot_product = 0.0
                # Vectorized dot product computation
                for j in range(dim):
                    dot_product += embeddings[i, j] * query[j]
                similarities[i] = dot_product

        # Alternative optimized kernel using tile-based computation
        @ti.kernel
        def compute_similarities_tiled(
            query: ti.types.ndarray(dtype=ti.f32),
            embeddings: ti.types.ndarray(dtype=ti.f32),
            similarities: ti.types.ndarray(dtype=ti.f32),
            n: ti.i32,
            dim: ti.i32
        ):
            tile_size = 16  # Adjust based on hardware
            for i in range(n):
                dot_product = 0.0
                # Process in tiles for better cache utilization
                for jt in range(0, dim):
                    if jt % tile_size != 0:
                        continue
                    tile_sum = 0.0
                    for j in range(jt, ti.min(jt + tile_size, dim)):
                        tile_sum += embeddings[i, j] * query[j]
                    dot_product += tile_sum
                similarities[i] = dot_product

        # Choose the appropriate kernel based on dimension size
        if query_embedding.shape[0] >= 256:
            compute_similarities_tiled(
                query_norm.astype(np.float32),
                self.normalized_embeddings,
                similarities,
                n,
                query_embedding.shape[0]
            )
        else:
            compute_similarities_optimized(
                query_norm.astype(np.float32),
                self.normalized_embeddings,
                similarities,
                n,
                query_embedding.shape[0]
            )

        # Optimize top-k selection
        if k >= n:
            indices = np.argsort(-similarities)
        else:
            # Use partial sort for better performance when k < n
            indices = np.argpartition(-similarities, k)[:k]
            indices = indices[np.argsort(-similarities[indices])]

        # Filter results efficiently using vectorized operations
        mask = similarities[indices] >= min_similarity
        filtered_indices = indices[mask]
        return [self.chunks[idx] for idx in filtered_indices[:k]]

    def save(self) -> bytes:
        return pickle.dumps({
            'embeddings': self.embeddings,
            'chunks': self.chunks
        })

    def load(self, data: bytes) -> 'NumpyVectorStore':
        loaded = pickle.loads(data)
        self.embeddings = loaded['embeddings']
        self.chunks = loaded['chunks']
        return self

    def clear(self) -> None:
        self.embeddings = np.empty((0, 0))
        self.chunks = []
        self.normalized_embeddings = None

    def rebuild_index(self) -> None:
        pass  # No index to rebuild for numpy implementation

    def _normalize_vector(self, vector: np.ndarray) -> np.ndarray:
        """Normalize a single vector efficiently."""
        return vector / (np.linalg.norm(vector) + 1e-8)

    def _precompute_normalized_embeddings(self) -> None:
        """Pre-compute and cache normalized embeddings."""
        # Allocate output array
        self.normalized_embeddings = np.empty_like(self.embeddings, dtype=np.float32)

        # Normalize embeddings using Taichi
        batch_normalize(
            self.embeddings.astype(np.float32),
            self.normalized_embeddings,
            self.embeddings.shape[0],
            self.embeddings.shape[1]
        )
types
AbstractVectorStore

Bases: ABC

Abstract base class for vector stores

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
class AbstractVectorStore(ABC):
    """Abstract base class for vector stores"""

    @abstractmethod
    def add_embeddings(self, embeddings: np.ndarray, chunks: list[Chunk]) -> None:
        """Add embeddings and their corresponding chunks to the store"""
        pass

    @abstractmethod
    def search(self, query_embedding: np.ndarray, k: int = 5, min_similarity: float = 0.7) -> list[Chunk]:
        """Search for similar vectors"""
        pass

    @abstractmethod
    def save(self) -> bytes:
        """Save the vector store to disk"""
        pass

    @abstractmethod
    def load(self, data: bytes) -> 'AbstractVectorStore':
        """Load the vector store from disk"""
        pass

    @abstractmethod
    def clear(self) -> None:
        """Clear all data from the store"""
        pass

    @abstractmethod
    def rebuild_index(self) -> None:
        """Optional for faster searches"""
        pass
add_embeddings(embeddings, chunks) abstractmethod

Add embeddings and their corresponding chunks to the store

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
21
22
23
24
@abstractmethod
def add_embeddings(self, embeddings: np.ndarray, chunks: list[Chunk]) -> None:
    """Add embeddings and their corresponding chunks to the store"""
    pass
clear() abstractmethod

Clear all data from the store

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
41
42
43
44
@abstractmethod
def clear(self) -> None:
    """Clear all data from the store"""
    pass
load(data) abstractmethod

Load the vector store from disk

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
36
37
38
39
@abstractmethod
def load(self, data: bytes) -> 'AbstractVectorStore':
    """Load the vector store from disk"""
    pass
rebuild_index() abstractmethod

Optional for faster searches

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
46
47
48
49
@abstractmethod
def rebuild_index(self) -> None:
    """Optional for faster searches"""
    pass
save() abstractmethod

Save the vector store to disk

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
31
32
33
34
@abstractmethod
def save(self) -> bytes:
    """Save the vector store to disk"""
    pass
search(query_embedding, k=5, min_similarity=0.7) abstractmethod

Search for similar vectors

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
26
27
28
29
@abstractmethod
def search(self, query_embedding: np.ndarray, k: int = 5, min_similarity: float = 0.7) -> list[Chunk]:
    """Search for similar vectors"""
    pass
Chunk dataclass

Represents a chunk of text with its embedding and metadata

Source code in toolboxv2/mods/isaa/base/VectorStores/types.py
 8
 9
10
11
12
13
14
15
@dataclass(slots=True)
class Chunk:
    """Represents a chunk of text with its embedding and metadata"""
    text: str
    embedding: np.ndarray
    metadata: dict[str, Any]
    content_hash: str
    cluster_id: int | None = None

extras

adapter
LiteLLM LLM Interface Module

This module provides interfaces for interacting with LiteLLM's language models, including text generation and embedding capabilities.

Author: Lightrag Team Created: 2025-02-04 License: MIT License Version: 1.0.0

Change Log: - 1.0.0 (2025-02-04): Initial LiteLLM release * Ported OpenAI logic to use litellm async client * Updated error types and environment variable names * Preserved streaming and embedding support

Dependencies
  • litellm
  • numpy
  • pipmaster
  • Python >= 3.10
Usage

from llm_interfaces.litellm import logging

if not hasattr(logging, 'NONE'): logging.NONE = 100

import litellm_complete, litellm_embed

litellm_complete(prompt, system_prompt=None, history_messages=None, keyword_extraction=False, model_name='groq/gemma2-9b-it', **kwargs) async

Public completion interface using the model name specified in the global configuration. Optionally extracts keywords if requested.

Source code in toolboxv2/mods/isaa/extras/adapter.py
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
async def litellm_complete(
    prompt, system_prompt=None, history_messages=None, keyword_extraction=False, model_name = "groq/gemma2-9b-it", **kwargs
) -> str | AsyncIterator[str]:
    """
    Public completion interface using the model name specified in the global configuration.
    Optionally extracts keywords if requested.
    """
    if history_messages is None:
        history_messages = []
    # Check and set response format for keyword extraction if needed
    keyword_extraction_flag = kwargs.pop("keyword_extraction", None)
    if keyword_extraction_flag:
        kwargs["response_format"] = "json"

    if "response_format" in kwargs:
        if isinstance(kwargs["response_format"], dict):
            kwargs["response_format"] = enforce_no_additional_properties(kwargs["response_format"])
        elif isinstance(kwargs["response_format"], str):
            pass
        else:
            kwargs["response_format"] = enforce_no_additional_properties(kwargs["response_format"].model_json_schema())  # oder .schema() in v1
     # kwargs["hashing_kv"].global_config["llm_model_name"]

    if any(x in model_name for x in ["mistral", "mixtral"]):
        kwargs.pop("response_format", None)

    return await litellm_complete_if_cache(
        model_name,
        prompt,
        system_prompt=system_prompt,
        history_messages=history_messages,
        **kwargs,
    )
litellm_complete_if_cache(model, prompt, system_prompt=None, history_messages=None, base_url=None, api_key=None, **kwargs) async

Core function to query the LiteLLM model. It builds the message context, invokes the completion API, and returns either a complete result string or an async iterator for streaming responses.

Source code in toolboxv2/mods/isaa/extras/adapter.py
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=4, max=10),
    retry=retry_if_exception_type((RateLimitError, Timeout, APIConnectionError)),
)
async def litellm_complete_if_cache(
    model,
    prompt,
    system_prompt=None,
    history_messages=None,
    base_url=None,
    api_key=None,
    **kwargs,
) -> str | AsyncIterator[str]:
    """
    Core function to query the LiteLLM model. It builds the message context,
    invokes the completion API, and returns either a complete result string or
    an async iterator for streaming responses.
    """
    # Set the API key if provided
    if api_key:
        os.environ["LITELLM_API_KEY"] = api_key

    # Remove internal keys not needed for the client call
    kwargs.pop("hashing_kv", None)
    kwargs.pop("keyword_extraction", None)

    fallbacks_ = kwargs.pop("fallbacks", [])
    # Build the messages list from system prompt, conversation history, and the new prompt
    messages = []
    if system_prompt:
        messages.append({"role": "system", "content": system_prompt})
    if history_messages is not None:
        messages.extend(history_messages)
    messages.append({"role": "user", "content": prompt})

    # Log query details for debugging purposes
    try:
        # Depending on the response format, choose the appropriate API call
        if "response_format" in kwargs:
            response = await acompletion(
                model=model, messages=messages,
                fallbacks=fallbacks_+os.getenv("FALLBACKS_MODELS", '').split(','),
                **kwargs
            )
        else:
            response = await acompletion(
                model=model, messages=messages,
                fallbacks=os.getenv("FALLBACKS_MODELS", '').split(','),
                **kwargs
            )
    except Exception as e:
        print(f"\n{model=}\n{prompt=}\n{system_prompt=}\n{history_messages=}\n{base_url=}\n{api_key=}\n{kwargs=}")
        get_logger().error(f"Failed to litellm memory work {e}")
        return ""

    # Check if the response is a streaming response (i.e. an async iterator)
    if hasattr(response, "__aiter__"):

        async def inner():
            async for chunk in response:
                # Assume LiteLLM response structure is similar to OpenAI's
                content = chunk.choices[0].delta.content
                if content is None:
                    continue
                yield content

        return inner()
    else:
        # Non-streaming: extract and return the full content string

        content = response.choices[0].message.content
        if content is None:
            content = response.choices[0].message.tool_calls[0].function.arguments
        return content
litellm_embed(texts, model='gemini/text-embedding-004', dimensions=256, base_url=None, api_key=None) async

Generates embeddings for the given list of texts using LiteLLM.

Source code in toolboxv2/mods/isaa/extras/adapter.py
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
@retry(
    stop=stop_after_attempt(3),
    wait=wait_exponential(multiplier=1, min=4, max=60),
    retry=retry_if_exception_type((RateLimitError, Timeout, APIConnectionError)),
)
async def litellm_embed(
    texts: list[str],
    model: str = "gemini/text-embedding-004",
    dimensions: int = 256,
    base_url: str = None,
    api_key: str = None,
) -> np.ndarray:
    """
    Generates embeddings for the given list of texts using LiteLLM.
    """
    response = await litellm.aembedding(
        model=model, input=texts,
        dimensions=dimensions,
        # encoding_format="float"
    )
    return np.array([dp.embedding for dp in response.data])
cahin_printer
ChainPrinter

Custom printer for enhanced chain visualization and progress display

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
class ChainPrinter:
    """Custom printer for enhanced chain visualization and progress display"""

    def __init__(self, verbose: bool = True):
        self.verbose = verbose
        self.colors = {
            'success': '\033[92m',
            'error': '\033[91m',
            'warning': '\033[93m',
            'info': '\033[94m',
            'highlight': '\033[95m',
            'dim': '\033[2m',
            'bold': '\033[1m',
            'reset': '\033[0m'
        }

    def _colorize(self, text: str, color: str) -> str:
        return f"{self.colors.get(color, '')}{text}{self.colors['reset']}"

    def print_header(self, title: str, subtitle: str = None):
        """Print formatted header"""
        print(f"\n{self._colorize('═' * 60, 'highlight')}")
        print(f"{self._colorize(f'🔗 {title}', 'bold')}")
        if subtitle:
            print(f"{self._colorize(subtitle, 'dim')}")
        print(f"{self._colorize('═' * 60, 'highlight')}\n")

    def print_success(self, message: str):
        print(f"{self._colorize('✅ ', 'success')}{message}")

    def print_error(self, message: str):
        print(f"{self._colorize('❌ ', 'error')}{message}")

    def print_warning(self, message: str):
        print(f"{self._colorize('⚠️ ', 'warning')}{message}")

    def print_info(self, message: str):
        print(f"{self._colorize('ℹ️ ', 'info')}{message}")

    def print_progress_start(self, chain_name: str):
        print(f"\n{self._colorize('🚀 Starting chain execution:', 'info')} {self._colorize(chain_name, 'bold')}")

    def print_task_start(self, task_name: str, current: int, total: int):
        progress = f"[{current + 1}/{total}]" if total > 0 else ""
        print(f"  {self._colorize('▶️ ', 'info')}{progress} {task_name}")

    def print_task_complete(self, task_name: str, completed: int, total: int):
        progress = f"[{completed}/{total}]" if total > 0 else ""
        print(f"  {self._colorize('✅', 'success')} {progress} {task_name} completed")

    def print_task_error(self, task_name: str, error: str):
        print(f"  {self._colorize('❌', 'error')} {task_name} failed: {error}")

    def print_progress_end(self, chain_name: str, duration: float, success: bool):
        status = self._colorize('✅ COMPLETED', 'success') if success else self._colorize('❌ FAILED', 'error')
        print(f"\n{status} {chain_name} ({duration:.2f}s)\n")

    def print_tool_usage_success(self, tool_name: str, duration: float, is_meta_tool: bool = False, tool_args: dict[str, Any] = None):
        if is_meta_tool:
            print(f"  {self._colorize('🔧 ', 'info')}{tool_name} completed ({duration:.2f}s) {arguments_summary(tool_args)}")
        else:
            print(f"  {self._colorize('🔩 ', 'info')}{tool_name} completed ({duration:.2f}s) {arguments_summary(tool_args)}")

    def print_tool_usage_error(self, tool_name: str, error: str, is_meta_tool: bool = False):
        if is_meta_tool:
            print(f"  {self._colorize('🔧 ', 'error')}{tool_name} failed: {error}")
        else:
            print(f"  {self._colorize('🔩 ', 'error')}{tool_name} failed: {error}")

    def print_outline_created(self, outline: dict):
        for step in outline.get("steps", []):
            print(f"  {self._colorize('📖 ', 'info')}Step: {self._colorize(step.get('description', 'Unknown'), 'dim')}")

    def print_reasoning_loop(self, loop_data: dict):
        print(f"  {self._colorize('🧠 ', 'info')}Reasoning Loop #{loop_data.get('loop_number', '?')}")
        print(
            f"    {self._colorize('📖 ', 'info')}Outline Step: {loop_data.get('outline_step', 0)} of {loop_data.get('outline_total', 0)}")
        print(f"    {self._colorize('📚 ', 'info')}Context Size: {loop_data.get('context_size', 0)} entries")
        print(f"    {self._colorize('📋 ', 'info')}Task Stack: {loop_data.get('task_stack_size', 0)} items")
        print(f"    {self._colorize('🔄 ', 'info')}Recovery Attempts: {loop_data.get('auto_recovery_attempts', 0)}")
        print(f"    {self._colorize('📊 ', 'info')}Performance Metrics: {loop_data.get('performance_metrics', {})}")

    def print_chain_list(self, chains: list[tuple[str, ChainMetadata]]):
        """Print formatted list of available chains"""
        if not chains:
            self.print_info("No chains found. Use 'create' to build your first chain.")
            return

        self.print_header("Available Chains", f"Total: {len(chains)}")

        for name, meta in chains:
            # Status indicators
            indicators = []
            if meta.has_parallels:
                indicators.append(self._colorize("⚡", "highlight"))
            if meta.has_conditionals:
                indicators.append(self._colorize("🔀", "warning"))
            if meta.has_error_handling:
                indicators.append(self._colorize("🛡️", "info"))

            status_str = " ".join(indicators) if indicators else ""

            # Complexity color
            complexity_colors = {"simple": "success", "medium": "warning", "complex": "error"}
            complexity = self._colorize(meta.complexity, complexity_colors.get(meta.complexity, "info"))

            print(f"  {self._colorize(name, 'bold')} {status_str}")
            print(f"    {meta.description or 'No description'}")
            print(f"    {complexity}{meta.agent_count} agents • {meta.version}")
            if meta.tags:
                tags_str = " ".join([f"#{tag}" for tag in meta.tags])
                print(f"    {self._colorize(tags_str, 'dim')}")
            print()
print_chain_list(chains)

Print formatted list of available chains

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
def print_chain_list(self, chains: list[tuple[str, ChainMetadata]]):
    """Print formatted list of available chains"""
    if not chains:
        self.print_info("No chains found. Use 'create' to build your first chain.")
        return

    self.print_header("Available Chains", f"Total: {len(chains)}")

    for name, meta in chains:
        # Status indicators
        indicators = []
        if meta.has_parallels:
            indicators.append(self._colorize("⚡", "highlight"))
        if meta.has_conditionals:
            indicators.append(self._colorize("🔀", "warning"))
        if meta.has_error_handling:
            indicators.append(self._colorize("🛡️", "info"))

        status_str = " ".join(indicators) if indicators else ""

        # Complexity color
        complexity_colors = {"simple": "success", "medium": "warning", "complex": "error"}
        complexity = self._colorize(meta.complexity, complexity_colors.get(meta.complexity, "info"))

        print(f"  {self._colorize(name, 'bold')} {status_str}")
        print(f"    {meta.description or 'No description'}")
        print(f"    {complexity}{meta.agent_count} agents • {meta.version}")
        if meta.tags:
            tags_str = " ".join([f"#{tag}" for tag in meta.tags])
            print(f"    {self._colorize(tags_str, 'dim')}")
        print()
print_header(title, subtitle=None)

Print formatted header

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py
84
85
86
87
88
89
90
def print_header(self, title: str, subtitle: str = None):
    """Print formatted header"""
    print(f"\n{self._colorize('═' * 60, 'highlight')}")
    print(f"{self._colorize(f'🔗 {title}', 'bold')}")
    if subtitle:
        print(f"{self._colorize(subtitle, 'dim')}")
    print(f"{self._colorize('═' * 60, 'highlight')}\n")
ChainProgressTracker

Enhanced progress tracker for chain execution with live display

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
class ChainProgressTracker:
    """Enhanced progress tracker for chain execution with live display"""

    def __init__(self, chain_printer: 'ChainPrinter' = None):
        self.events: list[ProgressEvent] = []
        self.start_time = time.time()
        self.chain_printer = chain_printer or ChainPrinter()
        self.current_task = None
        self.task_count = 0
        self.completed_tasks = 0

    async def emit_event(self, event: ProgressEvent):
        """Emit progress event with live display updates"""
        self.events.append(event)

        if event.event_type == "chain_start":
            self.task_count = event.metadata.get("task_count", 0)
            self.chain_printer.print_progress_start(event.node_name)

        elif event.event_type == "task_start":
            self.current_task = event.node_name
            self.chain_printer.print_task_start(event.node_name, self.completed_tasks, self.task_count)

        elif event.event_type == "task_complete":
            if event.status == NodeStatus.COMPLETED:
                self.completed_tasks += 1
                self.chain_printer.print_task_complete(event.node_name, self.completed_tasks, self.task_count)
            elif event.status == NodeStatus.FAILED:
                self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))

        elif event.event_type == "chain_end":
            duration = time.time() - self.start_time
            self.chain_printer.print_progress_end(event.node_name, duration, event.status == NodeStatus.COMPLETED)

        elif event.event_type == "tool_call" and event.success == False:
            self.chain_printer.print_tool_usage_error(event.tool_name, event.metadata.get("error",
                                                                                          event.metadata.get("message",
                                                                                                             event.error_details.get(
                                                                                                                 "error",
                                                                                                                 "Unknown error"))))

        elif event.event_type == "tool_call" and event.success == True:
            self.chain_printer.print_tool_usage_success(event.tool_name, event.duration, event.is_meta_tool, event.tool_args)

        elif event.event_type == "outline_created":
            self.chain_printer.print_outline_created(event.metadata.get("outline", {}))

        elif event.event_type == "reasoning_loop":
            self.chain_printer.print_reasoning_loop(event.metadata)

        elif event.event_type == "task_error":
            self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))
emit_event(event) async

Emit progress event with live display updates

Source code in toolboxv2/mods/isaa/extras/cahin_printer.py
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
async def emit_event(self, event: ProgressEvent):
    """Emit progress event with live display updates"""
    self.events.append(event)

    if event.event_type == "chain_start":
        self.task_count = event.metadata.get("task_count", 0)
        self.chain_printer.print_progress_start(event.node_name)

    elif event.event_type == "task_start":
        self.current_task = event.node_name
        self.chain_printer.print_task_start(event.node_name, self.completed_tasks, self.task_count)

    elif event.event_type == "task_complete":
        if event.status == NodeStatus.COMPLETED:
            self.completed_tasks += 1
            self.chain_printer.print_task_complete(event.node_name, self.completed_tasks, self.task_count)
        elif event.status == NodeStatus.FAILED:
            self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))

    elif event.event_type == "chain_end":
        duration = time.time() - self.start_time
        self.chain_printer.print_progress_end(event.node_name, duration, event.status == NodeStatus.COMPLETED)

    elif event.event_type == "tool_call" and event.success == False:
        self.chain_printer.print_tool_usage_error(event.tool_name, event.metadata.get("error",
                                                                                      event.metadata.get("message",
                                                                                                         event.error_details.get(
                                                                                                             "error",
                                                                                                             "Unknown error"))))

    elif event.event_type == "tool_call" and event.success == True:
        self.chain_printer.print_tool_usage_success(event.tool_name, event.duration, event.is_meta_tool, event.tool_args)

    elif event.event_type == "outline_created":
        self.chain_printer.print_outline_created(event.metadata.get("outline", {}))

    elif event.event_type == "reasoning_loop":
        self.chain_printer.print_reasoning_loop(event.metadata)

    elif event.event_type == "task_error":
        self.chain_printer.print_task_error(event.node_name, event.metadata.get("error", "Unknown error"))
mcp_session_manager
MCPSessionManager

Manages persistent MCP sessions with automatic reconnection and parallel processing

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
class MCPSessionManager:
    """Manages persistent MCP sessions with automatic reconnection and parallel processing"""

    def __init__(self):
        self.sessions: dict[str, ClientSession] = {}
        self.connections: dict[str, Any] = {}
        self.capabilities_cache: dict[str, dict] = {}
        self.retry_count: dict[str, int] = {}
        self.max_retries = 3
        self.connection_timeout = 15.0  # 10 seconds timeout
        self.operation_timeout = 10.0  # 5 seconds for operations

    async def get_session_with_timeout(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Get session with timeout protection"""
        try:
            return await asyncio.wait_for(
                self.get_session(server_name, server_config),
                timeout=self.connection_timeout
            )
        except TimeoutError:
            eprint(f"MCP session creation timeout for {server_name}")
            return None

    async def get_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Get or create persistent MCP session with proper context management"""
        if server_name in self.sessions:
            try:
                # Test if session is still alive with timeout
                session = self.sessions[server_name]
                # Quick connectivity test
                await asyncio.wait_for(session.list_tools(), timeout=2.0)
                return session
            except Exception as e:
                wprint(f"MCP session {server_name} failed, recreating: {e}")
                # Clean up the old session
                if server_name in self.sessions:
                    del self.sessions[server_name]
                if server_name in self.connections:
                    del self.connections[server_name]

        return await self._create_session(server_name, server_config)

    async def _create_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Create new MCP session with improved error handling"""
        try:
            command = server_config.get('command')
            args = server_config.get('args', [])
            env = server_config.get('env', {})
            transport_type = server_config.get('transport', 'stdio')

            if not command:
                eprint(f"No command specified for MCP server {server_name}")
                return None

            iprint(f"Creating MCP session for {server_name} (transport: {transport_type})")

            session = None

            # Create connection based on transport type
            if transport_type == 'stdio':
                session = await self._create_stdio_session(server_name, command, args, env)
            elif transport_type in ['http', 'streamable-http']:
                session = await self._create_http_session(server_name, server_config)
            else:
                eprint(f"Unsupported transport type: {transport_type}")
                return None

            if session:
                self.sessions[server_name] = session
                self.retry_count[server_name] = 0
                iprint(f"✓ MCP session created successfully: {server_name}")
                return session

            return None

        except Exception as e:
            self.retry_count[server_name] = self.retry_count.get(server_name, 0) + 1
            if self.retry_count[server_name] <= self.max_retries:
                wprint(f"MCP session creation failed (attempt {self.retry_count[server_name]}/{self.max_retries}): {e}")
                await asyncio.sleep(1.0)  # Longer delay before retry
                return await self._create_session(server_name, server_config)
            else:
                eprint(f"✗ MCP session creation failed after {self.max_retries} attempts: {e}")
                return None

    async def _create_stdio_session(self, server_name: str, command: str, args: list[str], env: dict[str, str]) -> \
    ClientSession | None:
        """Create stdio MCP session with fixed async context handling"""
        try:
            from mcp import StdioServerParameters
            from mcp.client.stdio import stdio_client

            # Prepare environment
            process_env = os.environ.copy()
            process_env.update(env)

            server_params = StdioServerParameters(
                command=command,
                args=args,
                env=process_env
            )

            # Create the stdio client and session in a single task context
            stdio_connection = stdio_client(server_params)

            # Enter the context manager
            read_stream, write_stream = await stdio_connection.__aenter__()

            # Store the connection for cleanup later
            self.connections[server_name] = stdio_connection

            # Create session
            session = ClientSession(read_stream, write_stream)

            # Initialize session in the same context
            await session.__aenter__()
            await asyncio.wait_for(session.initialize(), timeout=self.connection_timeout)

            return session

        except Exception as e:
            eprint(f"Failed to create stdio session for {server_name}: {e}")
            # Cleanup on failure
            if server_name in self.connections:
                with contextlib.suppress(Exception):
                    await self.connections[server_name].__aexit__(None, None, None)
                del self.connections[server_name]
            return None

    async def _create_http_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
        """Create HTTP MCP session with timeout"""
        try:
            from mcp.client.streamable_http import streamablehttp_client

            url = server_config.get('url', f"http://localhost:{server_config.get('port', 8000)}/mcp")

            connection = streamablehttp_client(url)
            read_stream, write_stream, cleanup = await asyncio.wait_for(
                connection.__aenter__(),
                timeout=self.connection_timeout
            )

            session = ClientSession(read_stream, write_stream)
            await session.__aenter__()
            await asyncio.wait_for(
                session.initialize(),
                timeout=self.connection_timeout
            )

            self.connections[server_name] = connection
            return session

        except Exception as e:
            eprint(f"Failed to create HTTP session for {server_name}: {e}")
            return None

    async def extract_capabilities_with_timeout(self, session: ClientSession, server_name: str) -> dict[str, dict]:
        """Extract capabilities with timeout protection"""
        try:
            return await asyncio.wait_for(
                self.extract_capabilities(session, server_name),
                timeout=self.operation_timeout
            )
        except TimeoutError:
            eprint(f"Capability extraction timeout for {server_name}")
            return {'tools': {}, 'resources': {}, 'resource_templates': {}, 'prompts': {}, 'images': {}}

    async def extract_capabilities(self, session: ClientSession, server_name: str) -> dict[str, dict]:
        """Extract all capabilities from MCP session"""
        if server_name in self.capabilities_cache:
            return self.capabilities_cache[server_name]

        capabilities = {
            'tools': {},
            'resources': {},
            'resource_templates': {},
            'prompts': {},
            'images': {}
        }

        try:
            # Extract tools with individual timeouts
            try:
                tools_response = await asyncio.wait_for(session.list_tools(), timeout=3.0)
                for tool in tools_response.tools:
                    capabilities['tools'][tool.name] = {
                        'name': tool.name,
                        'description': tool.description or '',
                        'input_schema': tool.inputSchema,
                        'output_schema': getattr(tool, 'outputSchema', None),
                        'display_name': getattr(tool, 'title', tool.name)
                    }
            except TimeoutError:
                wprint(f"Tools extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract tools from {server_name}: {e}")

            # Extract resources with timeout
            try:
                resources_response = await asyncio.wait_for(session.list_resources(), timeout=3.0)
                for resource in resources_response.resources:
                    capabilities['resources'][str(resource.uri)] = {
                        'uri': str(resource.uri),
                        'name': resource.name or str(resource.uri),
                        'description': resource.description or '',
                        'mime_type': getattr(resource, 'mimeType', None)
                    }
            except TimeoutError:
                wprint(f"Resources extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract resources from {server_name}: {e}")

            # Extract resource templates with timeout
            try:
                templates_response = await asyncio.wait_for(session.list_resource_templates(), timeout=3.0)
                for template in templates_response.resourceTemplates:
                    capabilities['resource_templates'][template.uriTemplate] = {
                        'uri_template': template.uriTemplate,
                        'name': template.name or template.uriTemplate,
                        'description': template.description or ''
                    }
            except TimeoutError:
                wprint(f"Resource templates extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract resource templates from {server_name}: {e}")

            # Extract prompts with timeout
            try:
                prompts_response = await asyncio.wait_for(session.list_prompts(), timeout=3.0)
                for prompt in prompts_response.prompts:
                    capabilities['prompts'][prompt.name] = {
                        'name': prompt.name,
                        'description': prompt.description or '',
                        'arguments': [
                            {
                                'name': arg.name,
                                'description': arg.description or '',
                                'required': arg.required
                            } for arg in (prompt.arguments or [])
                        ]
                    }
            except TimeoutError:
                wprint(f"Prompts extraction timeout for {server_name}")
            except Exception as e:
                wprint(f"Failed to extract prompts from {server_name}: {e}")

            self.capabilities_cache[server_name] = capabilities

            total_caps = (len(capabilities['tools']) + len(capabilities['resources']) +
                          len(capabilities['resource_templates']) + len(capabilities['prompts']))
            iprint(f"✓ Extracted {total_caps} capabilities from {server_name}")

        except Exception as e:
            eprint(f"Failed to extract capabilities from {server_name}: {e}")

        return capabilities

    async def _cleanup_session(self, server_name: str):
        """Clean up a specific session with proper context management"""
        try:
            # Clean up session first
            if server_name in self.sessions:
                try:
                    session = self.sessions[server_name]
                    await asyncio.wait_for(session.__aexit__(None, None, None), timeout=2.0)
                except (TimeoutError, Exception) as e:
                    wprint(f"Session cleanup warning for {server_name}: {e}")
                finally:
                    del self.sessions[server_name]

            # Clean up connection
            if server_name in self.connections:
                try:
                    connection = self.connections[server_name]
                    await asyncio.wait_for(connection.__aexit__(None, None, None), timeout=2.0)
                except (TimeoutError, Exception) as e:
                    wprint(f"Connection cleanup warning for {server_name}: {e}")
                finally:
                    del self.connections[server_name]

            # Clear cache
            if server_name in self.capabilities_cache:
                del self.capabilities_cache[server_name]

            # Reset retry count
            if server_name in self.retry_count:
                del self.retry_count[server_name]

        except Exception as e:
            wprint(f"Cleanup error for {server_name}: {e}")

    async def cleanup_all(self):
        """Clean up all sessions with timeout and proper error handling"""
        cleanup_tasks = []
        for server_name in list(self.sessions.keys()):
            task = asyncio.create_task(self._cleanup_session(server_name))
            cleanup_tasks.append(task)

        if cleanup_tasks:
            try:
                # Use gather with return_exceptions=True to collect all results
                results = await asyncio.wait_for(
                    asyncio.gather(*cleanup_tasks, return_exceptions=True),
                    timeout=5.0
                )

                # Log any non-cancellation exceptions
                for i, result in enumerate(results):
                    if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
                        wprint(f"Error cleaning up MCP session: {result}")

            except (asyncio.TimeoutError, asyncio.CancelledError):
                # Handle timeout and cancellation
                wprint("MCP session cleanup timeout or cancelled")
                # Cancel all tasks
                for task in cleanup_tasks:
                    if not task.done():
                        task.cancel()

                # Give tasks a moment to cancel cleanly
                try:
                    await asyncio.wait(cleanup_tasks, timeout=1.0)
                except asyncio.CancelledError:
                    pass

            except Exception as e:
                wprint(f"Unexpected error during MCP session cleanup: {e}")
                # Cancel remaining tasks
                for task in cleanup_tasks:
                    if not task.done():
                        task.cancel()
cleanup_all() async

Clean up all sessions with timeout and proper error handling

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
async def cleanup_all(self):
    """Clean up all sessions with timeout and proper error handling"""
    cleanup_tasks = []
    for server_name in list(self.sessions.keys()):
        task = asyncio.create_task(self._cleanup_session(server_name))
        cleanup_tasks.append(task)

    if cleanup_tasks:
        try:
            # Use gather with return_exceptions=True to collect all results
            results = await asyncio.wait_for(
                asyncio.gather(*cleanup_tasks, return_exceptions=True),
                timeout=5.0
            )

            # Log any non-cancellation exceptions
            for i, result in enumerate(results):
                if isinstance(result, Exception) and not isinstance(result, asyncio.CancelledError):
                    wprint(f"Error cleaning up MCP session: {result}")

        except (asyncio.TimeoutError, asyncio.CancelledError):
            # Handle timeout and cancellation
            wprint("MCP session cleanup timeout or cancelled")
            # Cancel all tasks
            for task in cleanup_tasks:
                if not task.done():
                    task.cancel()

            # Give tasks a moment to cancel cleanly
            try:
                await asyncio.wait(cleanup_tasks, timeout=1.0)
            except asyncio.CancelledError:
                pass

        except Exception as e:
            wprint(f"Unexpected error during MCP session cleanup: {e}")
            # Cancel remaining tasks
            for task in cleanup_tasks:
                if not task.done():
                    task.cancel()
extract_capabilities(session, server_name) async

Extract all capabilities from MCP session

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
async def extract_capabilities(self, session: ClientSession, server_name: str) -> dict[str, dict]:
    """Extract all capabilities from MCP session"""
    if server_name in self.capabilities_cache:
        return self.capabilities_cache[server_name]

    capabilities = {
        'tools': {},
        'resources': {},
        'resource_templates': {},
        'prompts': {},
        'images': {}
    }

    try:
        # Extract tools with individual timeouts
        try:
            tools_response = await asyncio.wait_for(session.list_tools(), timeout=3.0)
            for tool in tools_response.tools:
                capabilities['tools'][tool.name] = {
                    'name': tool.name,
                    'description': tool.description or '',
                    'input_schema': tool.inputSchema,
                    'output_schema': getattr(tool, 'outputSchema', None),
                    'display_name': getattr(tool, 'title', tool.name)
                }
        except TimeoutError:
            wprint(f"Tools extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract tools from {server_name}: {e}")

        # Extract resources with timeout
        try:
            resources_response = await asyncio.wait_for(session.list_resources(), timeout=3.0)
            for resource in resources_response.resources:
                capabilities['resources'][str(resource.uri)] = {
                    'uri': str(resource.uri),
                    'name': resource.name or str(resource.uri),
                    'description': resource.description or '',
                    'mime_type': getattr(resource, 'mimeType', None)
                }
        except TimeoutError:
            wprint(f"Resources extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract resources from {server_name}: {e}")

        # Extract resource templates with timeout
        try:
            templates_response = await asyncio.wait_for(session.list_resource_templates(), timeout=3.0)
            for template in templates_response.resourceTemplates:
                capabilities['resource_templates'][template.uriTemplate] = {
                    'uri_template': template.uriTemplate,
                    'name': template.name or template.uriTemplate,
                    'description': template.description or ''
                }
        except TimeoutError:
            wprint(f"Resource templates extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract resource templates from {server_name}: {e}")

        # Extract prompts with timeout
        try:
            prompts_response = await asyncio.wait_for(session.list_prompts(), timeout=3.0)
            for prompt in prompts_response.prompts:
                capabilities['prompts'][prompt.name] = {
                    'name': prompt.name,
                    'description': prompt.description or '',
                    'arguments': [
                        {
                            'name': arg.name,
                            'description': arg.description or '',
                            'required': arg.required
                        } for arg in (prompt.arguments or [])
                    ]
                }
        except TimeoutError:
            wprint(f"Prompts extraction timeout for {server_name}")
        except Exception as e:
            wprint(f"Failed to extract prompts from {server_name}: {e}")

        self.capabilities_cache[server_name] = capabilities

        total_caps = (len(capabilities['tools']) + len(capabilities['resources']) +
                      len(capabilities['resource_templates']) + len(capabilities['prompts']))
        iprint(f"✓ Extracted {total_caps} capabilities from {server_name}")

    except Exception as e:
        eprint(f"Failed to extract capabilities from {server_name}: {e}")

    return capabilities
extract_capabilities_with_timeout(session, server_name) async

Extract capabilities with timeout protection

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py
169
170
171
172
173
174
175
176
177
178
async def extract_capabilities_with_timeout(self, session: ClientSession, server_name: str) -> dict[str, dict]:
    """Extract capabilities with timeout protection"""
    try:
        return await asyncio.wait_for(
            self.extract_capabilities(session, server_name),
            timeout=self.operation_timeout
        )
    except TimeoutError:
        eprint(f"Capability extraction timeout for {server_name}")
        return {'tools': {}, 'resources': {}, 'resource_templates': {}, 'prompts': {}, 'images': {}}
get_session(server_name, server_config) async

Get or create persistent MCP session with proper context management

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
async def get_session(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
    """Get or create persistent MCP session with proper context management"""
    if server_name in self.sessions:
        try:
            # Test if session is still alive with timeout
            session = self.sessions[server_name]
            # Quick connectivity test
            await asyncio.wait_for(session.list_tools(), timeout=2.0)
            return session
        except Exception as e:
            wprint(f"MCP session {server_name} failed, recreating: {e}")
            # Clean up the old session
            if server_name in self.sessions:
                del self.sessions[server_name]
            if server_name in self.connections:
                del self.connections[server_name]

    return await self._create_session(server_name, server_config)
get_session_with_timeout(server_name, server_config) async

Get session with timeout protection

Source code in toolboxv2/mods/isaa/extras/mcp_session_manager.py
25
26
27
28
29
30
31
32
33
34
async def get_session_with_timeout(self, server_name: str, server_config: dict[str, Any]) -> ClientSession | None:
    """Get session with timeout protection"""
    try:
        return await asyncio.wait_for(
            self.get_session(server_name, server_config),
            timeout=self.connection_timeout
        )
    except TimeoutError:
        eprint(f"MCP session creation timeout for {server_name}")
        return None
modes
generate_prompt(subject, context='', additional_requirements=None)

Generates a prompt based on the given subject, with optional context and additional requirements.

Parameters: - subject (str): The main subject for the prompt. - context (str): Optional additional context to tailor the prompt. - additional_requirements (Dict[str, Any]): Optional additional parameters or requirements for the prompt.

Returns: - str: A crafted prompt.

Source code in toolboxv2/mods/isaa/extras/modes.py
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
def generate_prompt(subject: str, context: str = "", additional_requirements: dict[str, Any] = None) -> str:
    """
    Generates a prompt based on the given subject, with optional context and additional requirements.

    Parameters:
    - subject (str): The main subject for the prompt.
    - context (str): Optional additional context to tailor the prompt.
    - additional_requirements (Dict[str, Any]): Optional additional parameters or requirements for the prompt.

    Returns:
    - str: A crafted prompt.
    """
    prompt = f"Based on the subject '{subject}', with the context '{context}', generate a clear and precise instruction."
    if additional_requirements:
        prompt += f" Consider the following requirements: {additional_requirements}."
    return prompt
terminal_progress
AgentExecutionState

Verwaltet den gesamten Zustand des Agentenablaufs, um eine reichhaltige Visualisierung zu ermöglichen.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
class AgentExecutionState:
    """
    Verwaltet den gesamten Zustand des Agentenablaufs, um eine reichhaltige
    Visualisierung zu ermöglichen.
    """

    def __init__(self):
        self.agent_name = "Agent"
        self.execution_phase = 'initializing'
        self.start_time = time.time()
        self.error_count = 0
        self.outline = None
        self.outline_progress = {'current_step': 0, 'total_steps': 0}
        self.reasoning_notes = []
        self.current_reasoning_loop = 0
        self.active_delegation = None
        self.active_task_plan = None
        self.tool_history = []
        self.llm_interactions = {'total_calls': 0, 'total_cost': 0.0, 'total_tokens': 0}
        self.active_nodes = set()
        self.node_flow = []
        self.last_event_per_node = {}
        self.event_count = 0
ProgressiveTreePrinter

Eine moderne, produktionsreife Terminal-Visualisierung für den Agenten-Ablauf.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
class ProgressiveTreePrinter:
    """Eine moderne, produktionsreife Terminal-Visualisierung für den Agenten-Ablauf."""

    def __init__(self, **kwargs):
        self.processor = StateProcessor()
        self.style = Style()
        self.llm_stream_chunks = ""
        self.buffer = 0
        self._display_interval = 0.1
        self._last_update_time = time.time()
        self._terminal_width = 80
        self._terminal_height = 24
        self._is_initialized = False

        # Terminal-Größe ermitteln
        self._update_terminal_size()

        # Original print sichern
        import builtins
        self._original_print = builtins.print
        builtins.print = self.print
        self._terminal_content = []  # List für O(1) append


    def print(self, *args, **kwargs):
        """
        Überladene print Funktion die automatisch Content speichert
        """
        # Capture output in StringIO für Effizienz
        output = StringIO()
        if 'file' in kwargs:
            del kwargs['file']
        self._original_print(*args, file=output, **kwargs)
        content = output.getvalue()

        # Speichere nur wenn content nicht leer
        if content.strip():
            self._terminal_content.append(content.rstrip('\n'))

        # Normale Ausgabe
        self._original_print(*args, **kwargs)

    def live_print(self,*args, **kwargs):
        """
        Live print ohne Content-Speicherung für temporäre Ausgaben
        """
        self._original_print(*args, **kwargs)

    @staticmethod
    def clear():
        """
        Speichert aktuellen Terminal-Content und cleared das Terminal
        Systemagnostisch (Windows/Unix)
        """
        # Clear terminal - systemagnostisch
        if os.name == 'nt':  # Windows
            os.system('cls')
        else:  # Unix/Linux/macOS
            os.system('clear')

    def restore_content(self):
        """
        Stellt den gespeicherten Terminal-Content in einer Aktion wieder her
        Effizient durch join operation
        """
        if self._terminal_content:
            # Effiziente Wiederherstellung mit join
            restored_output = '\n'.join(self._terminal_content)
            self._original_print(restored_output)

    def _update_terminal_size(self):
        """Aktualisiert die Terminal-Dimensionen."""
        try:
            terminal_size = shutil.get_terminal_size()
            self._terminal_width = max(terminal_size.columns, 80)
            self._terminal_height = max(terminal_size.lines, 24)
        except:
            self._terminal_width = 80
            self._terminal_height = 24

    def _truncate_text(self, text: str, max_length: int) -> str:
        """Kürzt Text auf maximale Länge und fügt '...' hinzu."""
        if len(remove_styles(text)) <= max_length:
            return text

        # Berücksichtige Style-Codes beim Kürzen
        plain_text = remove_styles(text)
        if len(plain_text) > max_length - 3:
            truncated = plain_text[:max_length - 3] + "..."
            return truncated
        return text

    def _fit_content_to_terminal(self, lines: list) -> list:
        """Passt den Inhalt an die Terminal-Größe an."""
        fitted_lines = []
        available_width = self._terminal_width - 2  # Rand lassen

        for line in lines:
            if len(remove_styles(line)) > available_width:
                fitted_lines.append(self._truncate_text(line, available_width))
            else:
                fitted_lines.append(line)

        # Wenn zu viele Zeilen, die wichtigsten behalten
        max_lines = self._terminal_height - 3  # Platz für Header und Eingabezeile
        if len(fitted_lines) > max_lines:
            # Header behalten, dann die letzten Zeilen
            header_lines = fitted_lines[:5]  # Erste 5 Zeilen (Header)
            remaining_lines = fitted_lines[5:]

            if len(header_lines) < max_lines:
                content_space = max_lines - len(header_lines)
                fitted_lines = header_lines + remaining_lines[-content_space:]
            else:
                fitted_lines = fitted_lines[:max_lines]

        return fitted_lines

    async def progress_callback(self, event: ProgressEvent):
        """Haupteingangspunkt für Progress Events."""
        if event.event_type == 'execution_start':
            self.processor = StateProcessor()
            self._is_initialized = True


        self.processor.process_event(event)

        # LLM Stream Handling
        if event.event_type == 'llm_stream_chunk':
            self.llm_stream_chunks += event.llm_output
            # Stream-Chunks auf vernünftige Größe begrenzen
            lines = self.llm_stream_chunks.replace('\\n', '\n').split('\n')
            if len(lines) > 8:
                self.llm_stream_chunks = '\n'.join(lines[-8:])
            self.buffer += 1
            if self.buffer > 5:
                self.buffer = 0
            else:
                return

        if event.event_type == 'llm_call':
            self.llm_stream_chunks = ""

        # Display nur bei wichtigen Events oder zeitbasiert aktualisieren
        should_update = (
            time.time() - self._last_update_time > self._display_interval or
            event.event_type in ['execution_complete', 'outline_created', 'plan_created', 'node_enter']
        )

        if should_update and self._is_initialized:
            self._update_display()
            self._last_update_time = time.time()


        if event.event_type in ['execution_complete', 'error']:
            self.restore_content()
            self.print_final_summary()

    def _update_display(self):
        """Aktualisiert die Anzeige im Terminal."""
        self._update_terminal_size()  # Terminal-Größe neu ermitteln
        output_lines = self._render_full_display()

        self.clear()
        self.live_print('\n'.join(output_lines))


    def _render_full_display(self) -> list:
        """Rendert die komplette Anzeige als Liste von Zeilen."""
        state = self.processor.state
        all_lines = []

        # Header
        header_lines = self._render_header(state).split('\n')
        all_lines.extend(header_lines)
        all_lines.append("")  # Leerzeile

        # Hauptinhalt basierend auf Ausführungsphase
        if state.outline:
            outline_content = self._render_outline_section(state)
            if outline_content:
                all_lines.extend(outline_content)
                all_lines.append("")

        reasoning_content = self._render_reasoning_section(state)
        if reasoning_content:
            all_lines.extend(reasoning_content)
            all_lines.append("")

        activity_content = self._render_activity_section(state)
        if activity_content:
            all_lines.extend(activity_content)
            all_lines.append("")

        if state.active_task_plan:
            plan_content = self._render_task_plan_section(state)
            if plan_content:
                all_lines.extend(plan_content)
                all_lines.append("")

        if state.tool_history:
            tool_content = self._render_tool_history_section(state)
            if tool_content:
                all_lines.extend(tool_content)
                all_lines.append("")

        system_content = self._render_system_flow_section(state)
        if system_content:
            all_lines.extend(system_content)

        # An Terminal-Größe anpassen
        return self._fit_content_to_terminal(all_lines)

    def _render_header(self, state: AgentExecutionState) -> str:
        """Rendert den Header."""
        runtime = human_readable_time(time.time() - state.start_time)
        title = self.style.Bold(f"🤖 {state.agent_name}")
        phase = self.style.CYAN(state.execution_phase.upper())
        health_color = self.style.GREEN if state.error_count == 0 else self.style.YELLOW
        health = health_color(f"Fehler: {state.error_count}")

        header_line = f"{title} [{phase}] | {health} | ⏱️ {runtime}"
        separator = self.style.GREY("═" * min(len(remove_styles(header_line)), self._terminal_width - 2))

        return f"{header_line}\n{separator}"

    def _render_outline_section(self, state: AgentExecutionState) -> list:
        """Rendert die Outline-Sektion."""
        outline = state.outline
        progress = state.outline_progress
        if not outline or not outline.get('steps'):
            return []

        lines = [self.style.Bold(self.style.YELLOW("📋 Agenten-Plan"))]

        for i, step in enumerate(outline['steps'][:5], 1):  # Nur erste 5 Schritte
            status_icon = "⏸️"
            line_style = self.style.GREY

            if i < progress['current_step']:
                status_icon = "✅"
                line_style = self.style.GREEN
            elif i == progress['current_step']:
                status_icon = "🔄"
                line_style = self.style.Bold

            desc = step.get('description', f'Schritt {i}')[:60]  # Beschreibung kürzen
            method = self.style.CYAN(f"({step.get('method', 'N/A')})")

            lines.append(line_style(f"  {status_icon} Schritt {i}: {desc} {method}"))

        if len(outline['steps']) > 5:
            lines.append(self.style.GREY(f"  ... und {len(outline['steps']) - 5} weitere Schritte"))

        return lines

    def _render_reasoning_section(self, state: AgentExecutionState) -> list:
        """Rendert die Reasoning-Sektion."""
        notes = state.reasoning_notes
        if not notes:
            return []

        lines = [self.style.Bold(self.style.YELLOW("🧠 Denkprozess"))]

        # Nur die neueste Notiz anzeigen
        note = notes[-1]
        thought = note.get('thought', '...')[:100]  # Gedanken kürzen
        lines.append(f"  💭 {thought}")

        if note.get('current_focus'):
            focus = note['current_focus'][:80]
            lines.append(f"  🎯 Fokus: {self.style.CYAN(focus)}")

        if note.get('confidence_level') is not None:
            confidence = note['confidence_level']
            lines.append(f"  📊 Zuversicht: {self.style.YELLOW(f'{confidence:.0%}')}")

        if note.get('key_insights'):
            lines.append(f"  💡 Erkenntnisse:")
            for insight in note['key_insights'][:2]:  # Nur erste 2 Erkenntnisse
                insight_text = insight[:70]
                lines.append(f"    • {self.style.GREY(insight_text)}")

        return lines

    def _render_activity_section(self, state: AgentExecutionState) -> list:
        """Rendert die aktuelle Aktivität."""
        lines = [self.style.Bold(self.style.YELLOW(f"🔄 Aktivität (Loop {state.current_reasoning_loop})"))]

        if state.active_delegation:
            delegation = state.active_delegation

            if delegation['type'] == 'plan_creation':
                desc = delegation['description'][:80]
                lines.append(f"  📝 {desc}")

                if delegation.get('goals'):
                    lines.append(f"  🎯 Ziele: {len(delegation['goals'])}")
                    for goal in delegation['goals'][:2]:  # Nur erste 2 Ziele
                        goal_text = goal[:60]
                        lines.append(f"    • {self.style.GREY(goal_text)}")

            elif delegation['type'] == 'tool_delegation':
                desc = delegation['description'][:80]
                lines.append(f"  🛠️ {desc}")
                status = delegation.get('status', 'unbekannt')
                lines.append(f"  📊 Status: {self.style.CYAN(status)}")

                if delegation.get('tools'):
                    tools_text = ', '.join(delegation['tools'][:3])  # Nur erste 3 Tools
                    lines.append(f"  🔧 Tools: {tools_text}")

        # LLM-Statistiken kompakt
        llm = state.llm_interactions
        if llm['total_calls'] > 0:
            cost = f"${llm['total_cost']:.3f}"
            lines.append(
                self.style.GREY(f"  🤖 LLM: {llm['total_calls']} Calls | {cost} | {llm['total_tokens']:,} Tokens"))

        # LLM Stream (gekürzt)
        if self.llm_stream_chunks:
            stream_lines = self.llm_stream_chunks.splitlines()[-8:]
            for stream_line in stream_lines:
                truncated = stream_line[:self._terminal_width - 6]
                lines.append(self.style.GREY(f"  💬 {truncated}"))

        return lines

    def _render_task_plan_section(self, state: AgentExecutionState) -> list:
        """Rendert den Task-Plan kompakt."""
        plan: TaskPlan = state.active_task_plan
        if not plan:
            return []

        lines = [self.style.Bold(self.style.YELLOW(f"⚙️ Plan: {plan.name}"))]

        # Nur aktive und wichtige Tasks anzeigen
        sorted_tasks = sorted(plan.tasks, key=lambda t: (
            0 if t.status == 'running' else
            1 if t.status == 'failed' else
            2 if t.status == 'pending' else 3,
            getattr(t, 'priority', 99),
            t.id
        ))

        displayed_count = 0
        max_display = 5

        for task in sorted_tasks:
            if displayed_count >= max_display:
                remaining = len(sorted_tasks) - displayed_count
                lines.append(self.style.GREY(f"  ... und {remaining} weitere Tasks"))
                break

            icon = {"pending": "⏳", "running": "🔄", "completed": "✅", "failed": "❌"}.get(task.status, "❓")
            style_func = {"pending": self.style.GREY, "running": self.style.WHITE,
                          "completed": self.style.GREEN, "failed": self.style.RED}.get(task.status, self.style.WHITE)

            desc = task.description[:50]  # Beschreibung kürzen
            lines.append(style_func(f"  {icon} {task.id}: {desc}"))

            # Fehler anzeigen wenn vorhanden
            if hasattr(task, 'error') and task.error:
                error_text = task.error[:self._terminal_width - 5]
                lines.append(self.style.RED(f"    🔥 {error_text}"))

            displayed_count += 1

        return lines

    def _render_tool_history_section(self, state: AgentExecutionState) -> list:
        """Rendert die Tool-Historie kompakt."""
        history = state.tool_history
        if not history:
            return []

        lines = [self.style.Bold(self.style.YELLOW("🛠️ Tool-Historie"))]

        # Nur die letzten 5 Tools
        for event in reversed(history[-5:]):
            icon = "✅" if event.success else "❌"
            style_func = self.style.GREEN if event.success else self.style.RED
            duration = f"({human_readable_time(event.node_duration)})" if event.node_duration else ""

            tool_line = f"  {icon} {event.tool_name} {duration} {arguments_summary(event.tool_args, self._terminal_width)}"
            lines.append(style_func(tool_line))

            # Fehler kurz anzeigen
            if not event.success and event.tool_error:
                error_text = event.tool_error[:self._terminal_width - 5]
                lines.append(self.style.RED(f"    💥 {error_text}"))

        return lines

    def _render_system_flow_section(self, state: AgentExecutionState) -> list:
        """Rendert den System-Flow kompakt."""
        if not state.node_flow:
            return []

        lines = [self.style.Bold(self.style.YELLOW("🔧 System-Ablauf"))]

        # Nur aktive Nodes und die letzten paar
        recent_nodes = state.node_flow[-4:]  # Letzte 4 Nodes

        for i, node_name in enumerate(recent_nodes):
            is_last = (i == len(recent_nodes) - 1)
            prefix = "└─" if is_last else "├─"
            is_active = node_name in state.active_nodes
            icon = "🔄" if is_active else "✅"
            style_func = self.style.Bold if is_active else self.style.GREEN

            node_display = node_name[:30]  # Node-Namen kürzen
            lines.append(style_func(f"  {prefix} {icon} {node_display}"))

            # Aktive Node Details
            if is_active:
                last_event = state.last_event_per_node.get(node_name)
                if last_event and last_event.event_type == 'tool_call' and last_event.status == NodeStatus.RUNNING:
                    tool_name = last_event.tool_name[:25]
                    child_prefix = "     " if is_last else "  │  "
                    lines.append(self.style.GREY(f"{child_prefix}🔧 {tool_name}"))

        if len(state.node_flow) > 4:
            lines.append(self.style.GREY(f"  ... und {len(state.node_flow) - 4} weitere Nodes"))

        return lines

    def print_final_summary(self):
        """Zeigt die finale Zusammenfassung."""
        self._update_terminal_size()  # Terminal-Größe neu ermitteln
        output_lines = self._render_full_display()
        print('\n'.join(output_lines))
        summary_lines = [
            "",
            self.style.GREEN2(self.style.Bold("🏁 Ausführung Abgeschlossen")),
            self.style.GREY(f"Events verarbeitet: {self.processor.state.event_count}"),
            self.style.GREY(f"Gesamtlaufzeit: {human_readable_time(time.time() - self.processor.state.start_time)}"),
            ""
        ]

        for line in summary_lines:
            print(line)
clear() staticmethod

Speichert aktuellen Terminal-Content und cleared das Terminal Systemagnostisch (Windows/Unix)

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
389
390
391
392
393
394
395
396
397
398
399
@staticmethod
def clear():
    """
    Speichert aktuellen Terminal-Content und cleared das Terminal
    Systemagnostisch (Windows/Unix)
    """
    # Clear terminal - systemagnostisch
    if os.name == 'nt':  # Windows
        os.system('cls')
    else:  # Unix/Linux/macOS
        os.system('clear')
live_print(*args, **kwargs)

Live print ohne Content-Speicherung für temporäre Ausgaben

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
383
384
385
386
387
def live_print(self,*args, **kwargs):
    """
    Live print ohne Content-Speicherung für temporäre Ausgaben
    """
    self._original_print(*args, **kwargs)
print(*args, **kwargs)

Überladene print Funktion die automatisch Content speichert

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
def print(self, *args, **kwargs):
    """
    Überladene print Funktion die automatisch Content speichert
    """
    # Capture output in StringIO für Effizienz
    output = StringIO()
    if 'file' in kwargs:
        del kwargs['file']
    self._original_print(*args, file=output, **kwargs)
    content = output.getvalue()

    # Speichere nur wenn content nicht leer
    if content.strip():
        self._terminal_content.append(content.rstrip('\n'))

    # Normale Ausgabe
    self._original_print(*args, **kwargs)
print_final_summary()

Zeigt die finale Zusammenfassung.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
def print_final_summary(self):
    """Zeigt die finale Zusammenfassung."""
    self._update_terminal_size()  # Terminal-Größe neu ermitteln
    output_lines = self._render_full_display()
    print('\n'.join(output_lines))
    summary_lines = [
        "",
        self.style.GREEN2(self.style.Bold("🏁 Ausführung Abgeschlossen")),
        self.style.GREY(f"Events verarbeitet: {self.processor.state.event_count}"),
        self.style.GREY(f"Gesamtlaufzeit: {human_readable_time(time.time() - self.processor.state.start_time)}"),
        ""
    ]

    for line in summary_lines:
        print(line)
progress_callback(event) async

Haupteingangspunkt für Progress Events.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
async def progress_callback(self, event: ProgressEvent):
    """Haupteingangspunkt für Progress Events."""
    if event.event_type == 'execution_start':
        self.processor = StateProcessor()
        self._is_initialized = True


    self.processor.process_event(event)

    # LLM Stream Handling
    if event.event_type == 'llm_stream_chunk':
        self.llm_stream_chunks += event.llm_output
        # Stream-Chunks auf vernünftige Größe begrenzen
        lines = self.llm_stream_chunks.replace('\\n', '\n').split('\n')
        if len(lines) > 8:
            self.llm_stream_chunks = '\n'.join(lines[-8:])
        self.buffer += 1
        if self.buffer > 5:
            self.buffer = 0
        else:
            return

    if event.event_type == 'llm_call':
        self.llm_stream_chunks = ""

    # Display nur bei wichtigen Events oder zeitbasiert aktualisieren
    should_update = (
        time.time() - self._last_update_time > self._display_interval or
        event.event_type in ['execution_complete', 'outline_created', 'plan_created', 'node_enter']
    )

    if should_update and self._is_initialized:
        self._update_display()
        self._last_update_time = time.time()


    if event.event_type in ['execution_complete', 'error']:
        self.restore_content()
        self.print_final_summary()
restore_content()

Stellt den gespeicherten Terminal-Content in einer Aktion wieder her Effizient durch join operation

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
401
402
403
404
405
406
407
408
409
def restore_content(self):
    """
    Stellt den gespeicherten Terminal-Content in einer Aktion wieder her
    Effizient durch join operation
    """
    if self._terminal_content:
        # Effiziente Wiederherstellung mit join
        restored_output = '\n'.join(self._terminal_content)
        self._original_print(restored_output)
StateProcessor

Verarbeitet ProgressEvents und aktualisiert den AgentExecutionState.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
class StateProcessor:
    """Verarbeitet ProgressEvents und aktualisiert den AgentExecutionState."""

    def __init__(self):
        self.state = AgentExecutionState()

    def process_event(self, event: ProgressEvent):
        self.state.event_count += 1
        if event.agent_name:
            self.state.agent_name = event.agent_name

        # System-Level Events
        if event.event_type == 'node_enter' and event.node_name:
            self.state.active_nodes.add(event.node_name)
            if event.node_name not in self.state.node_flow:
                self.state.node_flow.append(event.node_name)
        elif event.event_type == 'node_exit' and event.node_name:
            self.state.active_nodes.discard(event.node_name)
        elif event.event_type == 'error':
            self.state.error_count += 1

        if event.node_name:
            self.state.last_event_per_node[event.node_name] = event

        # Outline & Reasoning Events
        if event.event_type == 'outline_created' and isinstance(event.metadata.get('outline'), dict):
            self.state.execution_phase = 'planning'
            self.state.outline = event.metadata['outline']
            self.state.outline_progress['total_steps'] = len(self.state.outline.get('steps', []))

        elif event.event_type == 'reasoning_loop':
            self.state.execution_phase = 'reasoning'
            self.state.current_reasoning_loop = event.metadata.get('loop_number', 0)
            self.state.outline_progress['current_step'] = event.metadata.get('outline_step', 0) + 1
            self.state.active_delegation = None

        # Task Plan & Execution Events
        elif event.event_type == 'plan_created' and event.metadata.get('full_plan'):
            self.state.execution_phase = 'executing_plan'
            self.state.active_task_plan = event.metadata['full_plan']
            self.state.active_delegation = None

        elif event.event_type in ['task_start', 'task_complete', 'task_error']:
            self._update_task_plan_status(event)

        # Tool & LLM Events
        elif event.event_type == 'tool_call':
            if event.is_meta_tool:
                self._process_meta_tool_call(event)
            else:
                if event.status in [NodeStatus.COMPLETED, NodeStatus.FAILED]:
                    self.state.tool_history.append(event)
                    if len(self.state.tool_history) > 5:
                        self.state.tool_history.pop(0)

        elif event.event_type == 'llm_call' and event.success:
            llm = self.state.llm_interactions
            llm['total_calls'] += 1
            llm['total_cost'] += event.llm_cost or 0
            llm['total_tokens'] += event.llm_total_tokens or 0

        elif event.event_type == 'execution_complete':
            self.state.execution_phase = 'completed'

    def _process_meta_tool_call(self, event: ProgressEvent):
        args = event.tool_args or {}
        if event.status != NodeStatus.RUNNING:
            return

        if event.tool_name == 'internal_reasoning':
            note = {k: args.get(k) for k in ['thought', 'current_focus', 'key_insights', 'confidence_level']}
            self.state.reasoning_notes.append(note)
            if len(self.state.reasoning_notes) > 3:
                self.state.reasoning_notes.pop(0)

        elif event.tool_name == 'delegate_to_llm_tool_node':
            self.state.active_delegation = {
                'type': 'tool_delegation',
                'description': args.get('task_description', 'N/A'),
                'tools': args.get('tools_list', []),
                'status': 'running'
            }

        elif event.tool_name == 'create_and_execute_plan':
            self.state.active_delegation = {
                'type': 'plan_creation',
                'description': f"Erstelle Plan für {len(args.get('goals', []))} Ziele",
                'goals': args.get('goals', []),
                'status': 'planning'
            }

    def _update_task_plan_status(self, event: ProgressEvent):
        plan = self.state.active_task_plan
        if not plan or not hasattr(plan, 'tasks'):
            return

        for task in plan.tasks:
            if hasattr(task, 'id') and task.id == event.task_id:
                if event.event_type == 'task_start':
                    task.status = 'running'
                elif event.event_type == 'task_complete':
                    task.status = 'completed'
                    task.result = event.tool_result or (event.metadata or {}).get("result")
                elif event.event_type == 'task_error':
                    task.status = 'failed'
                    task.error = (event.error_details or {}).get('message', 'Unbekannter Fehler')
                break
arguments_summary(tool_args, max_length=50)

Creates a summary of the tool arguments for display purposes.

Parameters:

Name Type Description Default
tool_args dict[str, Any]

Dictionary containing tool arguments

required
max_length int

Maximum length for individual argument values in summary

50

Returns:

Type Description
str

Formatted string summary of the arguments

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
def arguments_summary(tool_args: dict[str, Any], max_length: int = 50) -> str:
    """
    Creates a summary of the tool arguments for display purposes.

    Args:
        tool_args: Dictionary containing tool arguments
        max_length: Maximum length for individual argument values in summary

    Returns:
        Formatted string summary of the arguments
    """

    if not tool_args:
        return "No arguments"

    return_str = ""

    # Handle different types of arguments
    for key, value in tool_args.items():
        # Format the key
        formatted_key = key.replace('_', ' ').title()

        # Handle different value types
        if value is None:
            formatted_value = "None"
        elif isinstance(value, bool):
            formatted_value = str(value)
        elif isinstance(value, (int, float)):
            formatted_value = str(value)
        elif isinstance(value, str):
            # Truncate long strings
            if len(value) > max_length:
                formatted_value = f'"{value[:max_length - 3]}..."'
            else:
                formatted_value = f'"{value}"'
        elif isinstance(value, list):
            if not value:
                formatted_value = "[]"
            elif len(value) == 1:
                item = value[0]
                if isinstance(item, str) and len(item) > max_length:
                    formatted_value = f'["{item[:max_length - 6]}..."]'
                else:
                    formatted_value = f'["{item}"]' if isinstance(item, str) else f'[{item}]'
            else:
                formatted_value = f"[{len(value)} items]"
        elif isinstance(value, dict):
            if not value:
                formatted_value = "{}"
            else:
                keys = list(value.keys())[:3]  # Show first 3 keys
                if len(value) <= 3:
                    formatted_value = f"{{{', '.join(keys)}}}"
                else:
                    formatted_value = f"{{{', '.join(keys)}, ...}} ({len(value)} keys)"
        else:
            # Fallback for other types
            str_value = str(value)
            if len(str_value) > max_length:
                formatted_value = f"{str_value[:max_length - 3]}..."
            else:
                formatted_value = str_value

        # Add to return string
        if return_str:
            return_str += ", "
        return_str += f"{formatted_key}: {formatted_value}"

    # Handle meta-tool specific summaries
    if "tool_name" in tool_args:
        tool_name = tool_args["tool_name"]

        if tool_name == "internal_reasoning":
            meta_summary = []
            if "thought_number" in tool_args and "total_thoughts" in tool_args:
                meta_summary.append(f"Thought {tool_args['thought_number']}/{tool_args['total_thoughts']}")
            if "current_focus" in tool_args and tool_args["current_focus"]:
                focus = tool_args["current_focus"]
                if len(focus) > 30:
                    focus = focus[:27] + "..."
                meta_summary.append(f"Focus: {focus}")
            if "confidence_level" in tool_args:
                meta_summary.append(f"Confidence: {tool_args['confidence_level']}")

            if meta_summary:
                return_str = f"Internal Reasoning - {', '.join(meta_summary)}"

        elif tool_name == "manage_internal_task_stack":
            action = tool_args.get("action", "unknown")
            task_desc = tool_args.get("task_description", "")
            if len(task_desc) > 40:
                task_desc = task_desc[:37] + "..."
            return_str = f"Task Stack - Action: {action.title()}, Task: {task_desc}"

        elif tool_name == "delegate_to_llm_tool_node":
            task_desc = tool_args.get("task_description", "")
            tools_count = len(tool_args.get("tools_list", []))
            if len(task_desc) > 40:
                task_desc = task_desc[:37] + "..."
            return_str = f"Delegate - Task: {task_desc}, Tools: {tools_count}"

        elif tool_name == "create_and_execute_plan":
            goals_count = len(tool_args.get("goals", []))
            return_str = f"Create Plan - Goals: {goals_count}"

        elif tool_name == "advance_outline_step":
            completed = tool_args.get("step_completed", False)
            next_focus = tool_args.get("next_step_focus", "")
            if len(next_focus) > 30:
                next_focus = next_focus[:27] + "..."
            return_str = f"Advance Step - Completed: {completed}, Next: {next_focus}"

        elif tool_name == "write_to_variables":
            scope = tool_args.get("scope", "unknown")
            key = tool_args.get("key", "")
            return_str = f"Write Variable - Scope: {scope}, Key: {key}"

        elif tool_name == "read_from_variables":
            scope = tool_args.get("scope", "unknown")
            key = tool_args.get("key", "")
            return_str = f"Read Variable - Scope: {scope}, Key: {key}"

        elif tool_name == "direct_response":
            final_answer = tool_args.get("final_answer", "")
            if len(final_answer) > 50:
                final_answer = final_answer[:47] + "..."
            return_str = f"Direct Response - Answer: {final_answer}"

    # Handle live tool specific summaries
    elif any(key in tool_args for key in ["code", "filepath", "package_name"]):
        if "code" in tool_args:
            code = tool_args["code"]
            code_preview = code.replace('\n', ' ').strip()
            if len(code_preview) > 40:
                code_preview = code_preview[:37] + "..."
            return_str = f"Execute Code - {code_preview}"

        elif "filepath" in tool_args:
            filepath = tool_args["filepath"]
            if "content" in tool_args:
                content_length = len(str(tool_args["content"]))
                return_str = f"File Operation - Path: {filepath}, Content: {content_length} chars"
            elif "old_content" in tool_args and "new_content" in tool_args:
                return_str = f"Replace in File - Path: {filepath}, Replace operation"
            else:
                return_str = f"File Operation - Path: {filepath}"

        elif "package_name" in tool_args:
            package = tool_args["package_name"]
            version = tool_args.get("version", "latest")
            return_str = f"Install Package - {package} ({version})"

    # Ensure we don't exceed reasonable length for the entire summary
    if len(return_str) > 200:
        return_str = return_str[:197] + "..."

    return return_str
human_readable_time(seconds)

Konvertiert Sekunden in ein menschlich lesbares Format.

Source code in toolboxv2/mods/isaa/extras/terminal_progress.py
26
27
28
29
30
31
32
33
34
35
36
37
38
39
def human_readable_time(seconds: float) -> str:
    """Konvertiert Sekunden in ein menschlich lesbares Format."""
    if seconds is None:
        return ""
    if seconds < 1:
        return f"{seconds * 1000:.0f}ms"
    seconds = int(seconds)
    if seconds < 60:
        return f"{seconds}s"
    minutes, seconds = divmod(seconds, 60)
    if minutes < 60:
        return f"{minutes}m {seconds}s"
    hours, minutes = divmod(minutes, 60)
    return f"{hours}h {minutes}m"
verbose_output
DynamicVerboseFormatter

Unified, dynamic formatter that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
class DynamicVerboseFormatter:
    """Unified, dynamic formatter that adapts to screen size"""

    def __init__(self, print_func=None, min_width: int = 40, max_width: int = 240):
        self.style = Style()
        self.print = print_func or print
        self.min_width = min_width
        self.max_width = max_width
        self._terminal_width = self._get_terminal_width()


    def get_git_info(self):
        """Checks for a git repo and returns its name and branch, or None."""
        try:
            # Check if we are in a git repository
            subprocess.check_output(['git', 'rev-parse', '--is-inside-work-tree'], stderr=subprocess.DEVNULL)

            # Get the repo name (root folder name)
            repo_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
                                                stderr=subprocess.DEVNULL).strip().decode('utf-8')
            repo_name = os.path.basename(repo_root)

            # Get the current branch name
            branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                                             stderr=subprocess.DEVNULL).strip().decode('utf-8')

            return repo_name, branch
        except (subprocess.CalledProcessError, FileNotFoundError):
            # This handles cases where 'git' is not installed or it's not a git repo
            return None

    def _get_terminal_width(self) -> int:
        """Get current terminal width with fallback"""
        try:
            width = shutil.get_terminal_size().columns
            return max(self.min_width, min(width - 2, self.max_width))
        except (OSError, AttributeError):
            return 80

    def _wrap_text(self, text: str, width: int = None) -> list[str]:
        """Wrap text to fit terminal width"""
        if width is None:
            width = self._terminal_width - 4  # Account for borders

        words = text.split()
        lines = []
        current_line = []
        current_length = 0

        for word in words:
            if current_length + len(word) + len(current_line) <= width:
                current_line.append(word)
                current_length += len(word)
            else:
                if current_line:
                    lines.append(' '.join(current_line))
                current_line = [word]
                current_length = len(word)

        if current_line:
            lines.append(' '.join(current_line))

        return lines

    def _create_border(self, char: str = "─", width: int = None) -> str:
        """Create a border line that fits the terminal"""
        if width is None:
            width = self._terminal_width
        return char * width

    def _center_text(self, text: str, width: int = None) -> str:
        """Center text within the given width"""
        if width is None:
            width = self._terminal_width

        # Remove ANSI codes for length calculation
        clean_text = self._strip_ansi(text)
        padding = max(0, (width - len(clean_text)) // 2)
        return " " * padding + text

    def _strip_ansi(self, text: str) -> str:
        """Remove ANSI escape codes for length calculation"""
        import re
        ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
        return ansi_escape.sub('', text)

    def print_header(self, text: str):
        """Print a dynamic header that adapts to screen size"""
        self._terminal_width = self._get_terminal_width()

        if self._terminal_width < 60:  # Tiny screen
            self.print()
            self.print(self.style.CYAN("=" * self._terminal_width))
            self.print(self.style.CYAN(self.style.Bold(text)))
            self.print(self.style.CYAN("=" * self._terminal_width))
        else:  # Regular/large screen
            border_width = min(len(text) + 2, self._terminal_width - 2)
            border = "─" * border_width

            self.print()
            self.print(self.style.CYAN(f"┌{border}┐"))
            self.print(self.style.CYAN(f"│ {self.style.Bold(text).center(border_width - 2)} │"))
            self.print(self.style.CYAN(f"└{border}┘"))
        self.print()

    def print_section(self, title: str, content: str):
        """Print a clean section with adaptive formatting"""
        self._terminal_width = self._get_terminal_width()

        # Title
        if self._terminal_width < 60:
            self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(title)}")
        else:
            self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(self.style.BLUE(title))}")

        # Content with proper wrapping
        for line in content.split('\n'):
            if line.strip():
                wrapped_lines = self._wrap_text(line.strip())
                for wrapped_line in wrapped_lines:
                    if self._terminal_width < 60:
                        self.print(f"  {wrapped_line}")
                    else:
                        self.print(f"  {self.style.GREY('│')} {wrapped_line}")
        self.print()

    def print_progress_bar(self, current: int, maximum: int, title: str = "Progress"):
        """Dynamic progress bar that adapts to screen size"""
        self._terminal_width = self._get_terminal_width()

        # Calculate bar width based on screen size
        if self._terminal_width < 60:
            bar_width = 10
            template = f"\r{title}: [{{}}] {current}/{maximum}"
        else:
            bar_width = min(30, self._terminal_width - 30)
            template = f"\r{self.style.CYAN(title)}: [{{}}] {current}/{maximum} ({current / maximum * 100:.1f}%)"

        progress = int((current / maximum) * bar_width)
        bar = "█" * progress + "░" * (bar_width - progress)

        self.print(template.format(bar), end='', flush=True)

    def print_state(self, state: str, details: dict[str, Any] = None) -> str:
        """Print current state with adaptive formatting"""
        self._terminal_width = self._get_terminal_width()

        state_colors = {
            'ACTION': self.style.GREEN2,
            'PROCESSING': self.style.YELLOW2,
            'BRAKE': self.style.RED2,
            'DONE': self.style.BLUE2,
            'ERROR': self.style.RED,
            'SUCCESS': self.style.GREEN,
            'INFO': self.style.CYAN
        }

        color_func = state_colors.get(state.upper(), self.style.WHITE2)

        if self._terminal_width < 60:
            # Compact format for small screens
            self.print(f"\n[{color_func(state)}]")
            result = f"\n[{state}]"
        else:
            # Full format for larger screens
            self.print(f"\n{self.style.Bold('State:')} {color_func(state)}")
            result = f"\nState: {state}"

        if details:
            for key, value in details.items():
                # Truncate long values on small screens
                if self._terminal_width < 60 and len(str(value)) > 30:
                    display_value = str(value)[:27] + "..."
                else:
                    display_value = str(value)

                if self._terminal_width < 60:
                    self.print(f"  {key}: {display_value}")
                    result += f"\n  {key}: {display_value}"
                else:
                    self.print(f"  {self.style.GREY('├─')} {self.style.CYAN(key)}: {display_value}")
                    result += f"\n  ├─ {key}: {display_value}"

        return result

    def print_code_block(self, code: str, language: str = "python"):
        """Print code with syntax awareness and proper formatting"""
        self._terminal_width = self._get_terminal_width()

        if self._terminal_width < 60:
            # Simple format for small screens
            self.print(f"\n{self.style.GREY('Code:')}")
            for line in code.split('\n'):
                self.print(f"  {line}")
        else:
            # Detailed format for larger screens
            self.print(f"\n{self.style.BLUE('┌─')} {self.style.YELLOW2(f'{language.upper()} Code')}")

            lines = code.split('\n')
            for i, line in enumerate(lines):
                if i == len(lines) - 1 and not line.strip():
                    continue

                # Wrap long lines
                if len(line) > self._terminal_width - 6:
                    wrapped = self._wrap_text(line, self._terminal_width - 6)
                    for j, wrapped_line in enumerate(wrapped):
                        prefix = "│" if j == 0 else "│"
                        self.print(f"{self.style.BLUE(prefix)} {wrapped_line}")
                else:
                    self.print(f"{self.style.BLUE('│')} {line}")

            self.print(f"{self.style.BLUE('└─')} {self.style.GREY('End of code block')}")

    def print_table(self, headers: list[str], rows: list[list[str]]):
        """Print a dynamic table that adapts to screen size"""
        self._terminal_width = self._get_terminal_width()

        if not rows:
            return

        # Calculate column widths
        all_data = [headers] + rows
        col_widths = []

        for col in range(len(headers)):
            max_width = max(len(str(row[col])) for row in all_data if col < len(row))
            col_widths.append(min(max_width, self._terminal_width // len(headers) - 2))

        # Adjust if total width exceeds terminal
        total_width = sum(col_widths) + len(headers) * 3 + 1
        if total_width > self._terminal_width:
            # Proportionally reduce column widths
            scale_factor = (self._terminal_width - len(headers) * 3 - 1) / sum(col_widths)
            col_widths = [max(8, int(w * scale_factor)) for w in col_widths]

        # Print table
        self._print_table_row(headers, col_widths, is_header=True)
        self._print_table_separator(col_widths)

        for row in rows:
            self._print_table_row(row, col_widths)

    def _print_table_row(self, row: list[str], widths: list[int], is_header: bool = False):
        """Helper method to print a table row"""
        formatted_cells = []
        for _i, (cell, width) in enumerate(zip(row, widths, strict=False)):
            cell_str = str(cell)
            if len(cell_str) > width:
                cell_str = cell_str[:width - 3] + "..."

            if is_header:
                formatted_cells.append(self.style.Bold(self.style.CYAN(cell_str.ljust(width))))
            else:
                formatted_cells.append(cell_str.ljust(width))

        self.print(f"│ {' │ '.join(formatted_cells)} │")

    def _print_table_separator(self, widths: list[int]):
        """Helper method to print table separator"""
        parts = ['─' * w for w in widths]
        self.print(f"├─{'─┼─'.join(parts)}─┤")

    async def process_with_spinner(self, message: str, coroutine):
        """Execute coroutine with adaptive spinner"""
        self._terminal_width = self._get_terminal_width()

        if self._terminal_width < 60:
            # Simple spinner for small screens
            spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
        else:
            # Detailed spinner for larger screens
            spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"

        # Truncate message if too long
        if len(message) > self._terminal_width - 10:
            display_message = message[:self._terminal_width - 13] + "..."
        else:
            display_message = message

        with Spinner(f"{self.style.CYAN('●')} {display_message}", symbols=spinner_symbols):
            return await coroutine

    def print_git_info(self) -> str | None:
        """Get current git branch with error handling"""
        try:
            result = subprocess.run(
                ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                capture_output=True, text=True, timeout=2
            )
            if result.returncode == 0 and result.stdout.strip():
                branch = result.stdout.strip()

                # Check for uncommitted changes
                status_result = subprocess.run(
                    ['git', 'status', '--porcelain'],
                    capture_output=True, text=True, timeout=1
                )
                dirty = "*" if status_result.stdout.strip() else ""

                git_info = f"{branch}{dirty}"
                self.print_info(f"Git: {git_info}")
                return git_info
        except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
            pass
        return None

    # Convenience methods with consistent styling
    def print_error(self, message: str):
        """Print error message with consistent formatting"""
        self.print(f"{self.style.RED('✗')} {self.style.RED(message)}")

    def print_success(self, message: str):
        """Print success message with consistent formatting"""
        self.print(f"{self.style.GREEN('✓')} {self.style.GREEN(message)}")

    def print_warning(self, message: str):
        """Print warning message with consistent formatting"""
        self.print(f"{self.style.YELLOW('⚠')} {self.style.YELLOW(message)}")

    def print_info(self, message: str):
        """Print info message with consistent formatting"""
        self.print(f"{self.style.CYAN('ℹ')} {self.style.CYAN(message)}")

    def print_debug(self, message: str):
        """Print debug message with consistent formatting"""
        self.print(f"{self.style.GREY('🐛')} {self.style.GREY(message)}")
get_git_info()

Checks for a git repo and returns its name and branch, or None.

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def get_git_info(self):
    """Checks for a git repo and returns its name and branch, or None."""
    try:
        # Check if we are in a git repository
        subprocess.check_output(['git', 'rev-parse', '--is-inside-work-tree'], stderr=subprocess.DEVNULL)

        # Get the repo name (root folder name)
        repo_root = subprocess.check_output(['git', 'rev-parse', '--show-toplevel'],
                                            stderr=subprocess.DEVNULL).strip().decode('utf-8')
        repo_name = os.path.basename(repo_root)

        # Get the current branch name
        branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
                                         stderr=subprocess.DEVNULL).strip().decode('utf-8')

        return repo_name, branch
    except (subprocess.CalledProcessError, FileNotFoundError):
        # This handles cases where 'git' is not installed or it's not a git repo
        return None
print_code_block(code, language='python')

Print code with syntax awareness and proper formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
def print_code_block(self, code: str, language: str = "python"):
    """Print code with syntax awareness and proper formatting"""
    self._terminal_width = self._get_terminal_width()

    if self._terminal_width < 60:
        # Simple format for small screens
        self.print(f"\n{self.style.GREY('Code:')}")
        for line in code.split('\n'):
            self.print(f"  {line}")
    else:
        # Detailed format for larger screens
        self.print(f"\n{self.style.BLUE('┌─')} {self.style.YELLOW2(f'{language.upper()} Code')}")

        lines = code.split('\n')
        for i, line in enumerate(lines):
            if i == len(lines) - 1 and not line.strip():
                continue

            # Wrap long lines
            if len(line) > self._terminal_width - 6:
                wrapped = self._wrap_text(line, self._terminal_width - 6)
                for j, wrapped_line in enumerate(wrapped):
                    prefix = "│" if j == 0 else "│"
                    self.print(f"{self.style.BLUE(prefix)} {wrapped_line}")
            else:
                self.print(f"{self.style.BLUE('│')} {line}")

        self.print(f"{self.style.BLUE('└─')} {self.style.GREY('End of code block')}")
print_debug(message)

Print debug message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
336
337
338
def print_debug(self, message: str):
    """Print debug message with consistent formatting"""
    self.print(f"{self.style.GREY('🐛')} {self.style.GREY(message)}")
print_error(message)

Print error message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
320
321
322
def print_error(self, message: str):
    """Print error message with consistent formatting"""
    self.print(f"{self.style.RED('✗')} {self.style.RED(message)}")
print_git_info()

Get current git branch with error handling

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
def print_git_info(self) -> str | None:
    """Get current git branch with error handling"""
    try:
        result = subprocess.run(
            ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
            capture_output=True, text=True, timeout=2
        )
        if result.returncode == 0 and result.stdout.strip():
            branch = result.stdout.strip()

            # Check for uncommitted changes
            status_result = subprocess.run(
                ['git', 'status', '--porcelain'],
                capture_output=True, text=True, timeout=1
            )
            dirty = "*" if status_result.stdout.strip() else ""

            git_info = f"{branch}{dirty}"
            self.print_info(f"Git: {git_info}")
            return git_info
    except (subprocess.TimeoutExpired, FileNotFoundError, subprocess.SubprocessError):
        pass
    return None
print_header(text)

Print a dynamic header that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
def print_header(self, text: str):
    """Print a dynamic header that adapts to screen size"""
    self._terminal_width = self._get_terminal_width()

    if self._terminal_width < 60:  # Tiny screen
        self.print()
        self.print(self.style.CYAN("=" * self._terminal_width))
        self.print(self.style.CYAN(self.style.Bold(text)))
        self.print(self.style.CYAN("=" * self._terminal_width))
    else:  # Regular/large screen
        border_width = min(len(text) + 2, self._terminal_width - 2)
        border = "─" * border_width

        self.print()
        self.print(self.style.CYAN(f"┌{border}┐"))
        self.print(self.style.CYAN(f"│ {self.style.Bold(text).center(border_width - 2)} │"))
        self.print(self.style.CYAN(f"└{border}┘"))
    self.print()
print_info(message)

Print info message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
332
333
334
def print_info(self, message: str):
    """Print info message with consistent formatting"""
    self.print(f"{self.style.CYAN('ℹ')} {self.style.CYAN(message)}")
print_progress_bar(current, maximum, title='Progress')

Dynamic progress bar that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
def print_progress_bar(self, current: int, maximum: int, title: str = "Progress"):
    """Dynamic progress bar that adapts to screen size"""
    self._terminal_width = self._get_terminal_width()

    # Calculate bar width based on screen size
    if self._terminal_width < 60:
        bar_width = 10
        template = f"\r{title}: [{{}}] {current}/{maximum}"
    else:
        bar_width = min(30, self._terminal_width - 30)
        template = f"\r{self.style.CYAN(title)}: [{{}}] {current}/{maximum} ({current / maximum * 100:.1f}%)"

    progress = int((current / maximum) * bar_width)
    bar = "█" * progress + "░" * (bar_width - progress)

    self.print(template.format(bar), end='', flush=True)
print_section(title, content)

Print a clean section with adaptive formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
def print_section(self, title: str, content: str):
    """Print a clean section with adaptive formatting"""
    self._terminal_width = self._get_terminal_width()

    # Title
    if self._terminal_width < 60:
        self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(title)}")
    else:
        self.print(f"\n{self.style.BLUE('●')} {self.style.Bold(self.style.BLUE(title))}")

    # Content with proper wrapping
    for line in content.split('\n'):
        if line.strip():
            wrapped_lines = self._wrap_text(line.strip())
            for wrapped_line in wrapped_lines:
                if self._terminal_width < 60:
                    self.print(f"  {wrapped_line}")
                else:
                    self.print(f"  {self.style.GREY('│')} {wrapped_line}")
    self.print()
print_state(state, details=None)

Print current state with adaptive formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
def print_state(self, state: str, details: dict[str, Any] = None) -> str:
    """Print current state with adaptive formatting"""
    self._terminal_width = self._get_terminal_width()

    state_colors = {
        'ACTION': self.style.GREEN2,
        'PROCESSING': self.style.YELLOW2,
        'BRAKE': self.style.RED2,
        'DONE': self.style.BLUE2,
        'ERROR': self.style.RED,
        'SUCCESS': self.style.GREEN,
        'INFO': self.style.CYAN
    }

    color_func = state_colors.get(state.upper(), self.style.WHITE2)

    if self._terminal_width < 60:
        # Compact format for small screens
        self.print(f"\n[{color_func(state)}]")
        result = f"\n[{state}]"
    else:
        # Full format for larger screens
        self.print(f"\n{self.style.Bold('State:')} {color_func(state)}")
        result = f"\nState: {state}"

    if details:
        for key, value in details.items():
            # Truncate long values on small screens
            if self._terminal_width < 60 and len(str(value)) > 30:
                display_value = str(value)[:27] + "..."
            else:
                display_value = str(value)

            if self._terminal_width < 60:
                self.print(f"  {key}: {display_value}")
                result += f"\n  {key}: {display_value}"
            else:
                self.print(f"  {self.style.GREY('├─')} {self.style.CYAN(key)}: {display_value}")
                result += f"\n  ├─ {key}: {display_value}"

    return result
print_success(message)

Print success message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
324
325
326
def print_success(self, message: str):
    """Print success message with consistent formatting"""
    self.print(f"{self.style.GREEN('✓')} {self.style.GREEN(message)}")
print_table(headers, rows)

Print a dynamic table that adapts to screen size

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
def print_table(self, headers: list[str], rows: list[list[str]]):
    """Print a dynamic table that adapts to screen size"""
    self._terminal_width = self._get_terminal_width()

    if not rows:
        return

    # Calculate column widths
    all_data = [headers] + rows
    col_widths = []

    for col in range(len(headers)):
        max_width = max(len(str(row[col])) for row in all_data if col < len(row))
        col_widths.append(min(max_width, self._terminal_width // len(headers) - 2))

    # Adjust if total width exceeds terminal
    total_width = sum(col_widths) + len(headers) * 3 + 1
    if total_width > self._terminal_width:
        # Proportionally reduce column widths
        scale_factor = (self._terminal_width - len(headers) * 3 - 1) / sum(col_widths)
        col_widths = [max(8, int(w * scale_factor)) for w in col_widths]

    # Print table
    self._print_table_row(headers, col_widths, is_header=True)
    self._print_table_separator(col_widths)

    for row in rows:
        self._print_table_row(row, col_widths)
print_warning(message)

Print warning message with consistent formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
328
329
330
def print_warning(self, message: str):
    """Print warning message with consistent formatting"""
    self.print(f"{self.style.YELLOW('⚠')} {self.style.YELLOW(message)}")
process_with_spinner(message, coroutine) async

Execute coroutine with adaptive spinner

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
async def process_with_spinner(self, message: str, coroutine):
    """Execute coroutine with adaptive spinner"""
    self._terminal_width = self._get_terminal_width()

    if self._terminal_width < 60:
        # Simple spinner for small screens
        spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
    else:
        # Detailed spinner for larger screens
        spinner_symbols = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"

    # Truncate message if too long
    if len(message) > self._terminal_width - 10:
        display_message = message[:self._terminal_width - 13] + "..."
    else:
        display_message = message

    with Spinner(f"{self.style.CYAN('●')} {display_message}", symbols=spinner_symbols):
        return await coroutine
EnhancedVerboseOutput

Main interface for verbose output with full functionality

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
class EnhancedVerboseOutput:
    """Main interface for verbose output with full functionality"""

    def __init__(self, verbose: bool = True, print_func=None, **formatter_kwargs):
        self.verbose = verbose
        self.print = print_func or print
        self.formatter = DynamicVerboseFormatter(self.print, **formatter_kwargs)
        self._start_time = time.time()

    def __getattr__(self, name):
        """Delegate to formatter for convenience"""
        return getattr(self.formatter, name)

    async def print_agent_response(self, response: str):
        await self.log_message("assistant", response)

    async def print_thought(self, thought: str):
        await self.log_message("assistant", f"Thought: {thought}")

    async def log_message(self, role: str, content: str):
        """Log chat messages with role-based formatting"""
        if not self.verbose:
            return

        role_formats = {
            'user': (self.formatter.style.GREEN, "👤"),
            'assistant': (self.formatter.style.BLUE, "🤖"),
            'system': (self.formatter.style.YELLOW, "⚙️"),
            'error': (self.formatter.style.RED, "❌"),
            'debug': (self.formatter.style.GREY, "🐛")
        }

        color_func, icon = role_formats.get(role.lower(), (self.formatter.style.WHITE, "•"))

        if content.startswith("```"):
            self.formatter.print_code_block(content)
            return

        if content.startswith("{") or content.startswith("[") and content.endswith("}") or content.endswith("]"):
            content = json.dumps(json.loads(content), indent=2)

        # Adapt formatting based on screen size
        if self.formatter._terminal_width < 60:
            self.print(f"\n{icon} [{role.upper()}]")
            # Wrap content for small screens
            wrapped_content = self.formatter._wrap_text(content, self.formatter._terminal_width - 2)
            for line in wrapped_content:
                self.print(f"  {line}")
        else:
            self.print(f"\n{icon} {color_func(f'[{role.upper()}]')}")
            self.print(f"{self.formatter.style.GREY('└─')} {content}")
        self.print()

    async def log_process_result(self, result: dict[str, Any]):
        """Log processing results with structured formatting"""
        if not self.verbose:
            return

        content_parts = []

        if 'action' in result:
            content_parts.append(f"Action: {result['action']}")
        if 'is_completed' in result:
            content_parts.append(f"Completed: {result['is_completed']}")
        if 'effectiveness' in result:
            content_parts.append(f"Effectiveness: {result['effectiveness']}")
        if 'recommendations' in result:
            content_parts.append(f"Recommendations:\n{result['recommendations']}")
        if 'workflow' in result:
            content_parts.append(f"Workflow:\n{result['workflow']}")
        if 'errors' in result and result['errors']:
            content_parts.append(f"Errors: {result['errors']}")
        if 'content' in result:
            content_parts.append(f"Content:\n{result['content']}")

        self.formatter.print_section("Process Result", '\n'.join(content_parts))

    def log_header(self, text: str):
        """Log header with timing information"""
        if not self.verbose:
            return

        elapsed = time.time() - self._start_time
        timing = f" ({elapsed / 60:.1f}m)" if elapsed > 60 else f" ({elapsed:.1f}s)"

        self.formatter.print_header(f"{text}{timing}")

    def log_state(self, state: str, user_ns: dict = None, override: bool = False):
        """Log state with optional override"""
        if not self.verbose and not override:
            return

        return self.formatter.print_state(state, user_ns)

    async def process(self, message: str, coroutine):
        """Process with optional spinner"""
        if not self.verbose:
            return await coroutine

        if message.lower() in ["code", "silent"]:
            return await coroutine

        return await self.formatter.process_with_spinner(message, coroutine)

    def print_tool_call(self, tool_name: str, tool_args: dict, result: str | None = None):
        """
        Gibt Informationen zum Tool-Aufruf aus.
        Versucht, das Ergebnis als JSON zu formatieren, wenn möglich.
        """
        if not self.verbose:
            return

        # Argumente wie zuvor formatieren
        args_str = json.dumps(tool_args, indent=2, ensure_ascii=False) if tool_args else "None"
        content = f"Tool: {tool_name}\nArguments:\n{args_str}"

        if result:
            result_output = ""
            try:
                # 1. Versuch, den String als JSON zu parsen
                data = json.loads(result)

                # 2. Prüfen, ob das Ergebnis ein Dictionary ist (der häufigste Fall)
                if isinstance(data, dict):
                    # Eine Kopie für die Anzeige erstellen, um den 'output'-Wert zu ersetzen
                    display_data = data.copy()
                    output_preview = ""

                    # Spezielle Handhabung für einen langen 'output'-String, falls vorhanden
                    if 'output' in display_data and isinstance(display_data['output'], str):
                        full_output = display_data['output']
                        # Den langen String im JSON durch einen Platzhalter ersetzen
                        display_data['output'] = "<-- [Inhalt wird separat formatiert]"

                        # Vorschau mit den ersten 3 Zeilen erstellen
                        lines = full_output.strip().split('\n')[:3]
                        preview_text = '\n'.join(lines)
                        output_preview = f"\n\n--- Vorschau für 'output' ---\n\x1b[90m{preview_text}\n...\x1b[0m"  # Hellgrauer Text
                        # display_data['output'] = output_preview
                    # Das formatierte JSON (mit Platzhalter) zum Inhalt hinzufügen
                    formatted_json = json.dumps(display_data, indent=2, ensure_ascii=False)
                    result_output = f"Geparstes Dictionary:\n{formatted_json}{output_preview}"

                else:
                    # Falls es valides JSON, aber kein Dictionary ist (z.B. eine Liste)
                    result_output = f"Gepastes JSON (kein Dictionary):\n{json.dumps(data, indent=2, ensure_ascii=False)}"

            except json.JSONDecodeError:
                # 3. Wenn Parsen fehlschlägt, den String als Rohtext behandeln
                result_output = f"{result}"

            content += f"\nResult:\n{result_output}"

        else:
            # Fall, wenn der Task noch läuft
            content += "\nResult: In progress..."

        # Den gesamten Inhalt an den Formatter übergeben
        self.formatter.print_section("Tool Call", content)

    def print_event(self, event: dict):
        """Print event information"""
        if not self.verbose:
            return

        if event.get("content") and event["content"].get("parts"):
            for part in event["content"]["parts"]:
                if part.get("text"):
                    self.formatter.print_info(f"Thought: {part['text']}")
                if part.get("function_call"):
                    self.print_tool_call(
                        part["function_call"]["name"],
                        part["function_call"]["args"]
                    )
                if part.get("function_response"):
                    result = part["function_response"]["response"].get("result", "")
                    self.print_tool_call(
                        part["function_response"]["name"],
                        {},
                        str(result)
                    )

        if event.get("usage_metadata"):
            self.formatter.print_info(f"Token usage: {event['usage_metadata']}")

    @contextmanager
    def section_context(self, title: str):
        """Context manager for sections"""
        if self.verbose:
            self.formatter.print_section(title, "Starting...")
        try:
            yield
        finally:
            if self.verbose:
                self.formatter.print_success(f"Completed: {title}")

    def clear_line(self):
        """Clear current line"""
        self.print('\r' + ' ' * self.formatter._terminal_width + '\r', end='')

    def print_separator(self, char: str = "─"):
        """Print a separator line"""
        self.print(self.formatter.style.GREY(char * self.formatter._terminal_width))

    def print_warning(self, message: str):
        """Print a warning message with yellow style"""
        if self.verbose:
            self.print(self.formatter.style.YELLOW(f"⚠️  WARNING: {message}"))

    def print_error(self, message: str):
        """Print an error message with red style"""
        if self.verbose:
            self.print(self.formatter.style.RED(f"❌ ERROR: {message}"))

    def print_success(self, message: str):
        """Print a success message with green style"""
        if self.verbose:
            self.print(self.formatter.style.GREEN(f"✅ SUCCESS: {message}"))
__getattr__(name)

Delegate to formatter for convenience

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
350
351
352
def __getattr__(self, name):
    """Delegate to formatter for convenience"""
    return getattr(self.formatter, name)
clear_line()

Clear current line

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
537
538
539
def clear_line(self):
    """Clear current line"""
    self.print('\r' + ' ' * self.formatter._terminal_width + '\r', end='')
log_header(text)

Log header with timing information

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
418
419
420
421
422
423
424
425
426
def log_header(self, text: str):
    """Log header with timing information"""
    if not self.verbose:
        return

    elapsed = time.time() - self._start_time
    timing = f" ({elapsed / 60:.1f}m)" if elapsed > 60 else f" ({elapsed:.1f}s)"

    self.formatter.print_header(f"{text}{timing}")
log_message(role, content) async

Log chat messages with role-based formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
async def log_message(self, role: str, content: str):
    """Log chat messages with role-based formatting"""
    if not self.verbose:
        return

    role_formats = {
        'user': (self.formatter.style.GREEN, "👤"),
        'assistant': (self.formatter.style.BLUE, "🤖"),
        'system': (self.formatter.style.YELLOW, "⚙️"),
        'error': (self.formatter.style.RED, "❌"),
        'debug': (self.formatter.style.GREY, "🐛")
    }

    color_func, icon = role_formats.get(role.lower(), (self.formatter.style.WHITE, "•"))

    if content.startswith("```"):
        self.formatter.print_code_block(content)
        return

    if content.startswith("{") or content.startswith("[") and content.endswith("}") or content.endswith("]"):
        content = json.dumps(json.loads(content), indent=2)

    # Adapt formatting based on screen size
    if self.formatter._terminal_width < 60:
        self.print(f"\n{icon} [{role.upper()}]")
        # Wrap content for small screens
        wrapped_content = self.formatter._wrap_text(content, self.formatter._terminal_width - 2)
        for line in wrapped_content:
            self.print(f"  {line}")
    else:
        self.print(f"\n{icon} {color_func(f'[{role.upper()}]')}")
        self.print(f"{self.formatter.style.GREY('└─')} {content}")
    self.print()
log_process_result(result) async

Log processing results with structured formatting

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
async def log_process_result(self, result: dict[str, Any]):
    """Log processing results with structured formatting"""
    if not self.verbose:
        return

    content_parts = []

    if 'action' in result:
        content_parts.append(f"Action: {result['action']}")
    if 'is_completed' in result:
        content_parts.append(f"Completed: {result['is_completed']}")
    if 'effectiveness' in result:
        content_parts.append(f"Effectiveness: {result['effectiveness']}")
    if 'recommendations' in result:
        content_parts.append(f"Recommendations:\n{result['recommendations']}")
    if 'workflow' in result:
        content_parts.append(f"Workflow:\n{result['workflow']}")
    if 'errors' in result and result['errors']:
        content_parts.append(f"Errors: {result['errors']}")
    if 'content' in result:
        content_parts.append(f"Content:\n{result['content']}")

    self.formatter.print_section("Process Result", '\n'.join(content_parts))
log_state(state, user_ns=None, override=False)

Log state with optional override

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
428
429
430
431
432
433
def log_state(self, state: str, user_ns: dict = None, override: bool = False):
    """Log state with optional override"""
    if not self.verbose and not override:
        return

    return self.formatter.print_state(state, user_ns)
print_error(message)

Print an error message with red style

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
550
551
552
553
def print_error(self, message: str):
    """Print an error message with red style"""
    if self.verbose:
        self.print(self.formatter.style.RED(f"❌ ERROR: {message}"))
print_event(event)

Print event information

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
def print_event(self, event: dict):
    """Print event information"""
    if not self.verbose:
        return

    if event.get("content") and event["content"].get("parts"):
        for part in event["content"]["parts"]:
            if part.get("text"):
                self.formatter.print_info(f"Thought: {part['text']}")
            if part.get("function_call"):
                self.print_tool_call(
                    part["function_call"]["name"],
                    part["function_call"]["args"]
                )
            if part.get("function_response"):
                result = part["function_response"]["response"].get("result", "")
                self.print_tool_call(
                    part["function_response"]["name"],
                    {},
                    str(result)
                )

    if event.get("usage_metadata"):
        self.formatter.print_info(f"Token usage: {event['usage_metadata']}")
print_separator(char='─')

Print a separator line

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
541
542
543
def print_separator(self, char: str = "─"):
    """Print a separator line"""
    self.print(self.formatter.style.GREY(char * self.formatter._terminal_width))
print_success(message)

Print a success message with green style

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
555
556
557
558
def print_success(self, message: str):
    """Print a success message with green style"""
    if self.verbose:
        self.print(self.formatter.style.GREEN(f"✅ SUCCESS: {message}"))
print_tool_call(tool_name, tool_args, result=None)

Gibt Informationen zum Tool-Aufruf aus. Versucht, das Ergebnis als JSON zu formatieren, wenn möglich.

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
def print_tool_call(self, tool_name: str, tool_args: dict, result: str | None = None):
    """
    Gibt Informationen zum Tool-Aufruf aus.
    Versucht, das Ergebnis als JSON zu formatieren, wenn möglich.
    """
    if not self.verbose:
        return

    # Argumente wie zuvor formatieren
    args_str = json.dumps(tool_args, indent=2, ensure_ascii=False) if tool_args else "None"
    content = f"Tool: {tool_name}\nArguments:\n{args_str}"

    if result:
        result_output = ""
        try:
            # 1. Versuch, den String als JSON zu parsen
            data = json.loads(result)

            # 2. Prüfen, ob das Ergebnis ein Dictionary ist (der häufigste Fall)
            if isinstance(data, dict):
                # Eine Kopie für die Anzeige erstellen, um den 'output'-Wert zu ersetzen
                display_data = data.copy()
                output_preview = ""

                # Spezielle Handhabung für einen langen 'output'-String, falls vorhanden
                if 'output' in display_data and isinstance(display_data['output'], str):
                    full_output = display_data['output']
                    # Den langen String im JSON durch einen Platzhalter ersetzen
                    display_data['output'] = "<-- [Inhalt wird separat formatiert]"

                    # Vorschau mit den ersten 3 Zeilen erstellen
                    lines = full_output.strip().split('\n')[:3]
                    preview_text = '\n'.join(lines)
                    output_preview = f"\n\n--- Vorschau für 'output' ---\n\x1b[90m{preview_text}\n...\x1b[0m"  # Hellgrauer Text
                    # display_data['output'] = output_preview
                # Das formatierte JSON (mit Platzhalter) zum Inhalt hinzufügen
                formatted_json = json.dumps(display_data, indent=2, ensure_ascii=False)
                result_output = f"Geparstes Dictionary:\n{formatted_json}{output_preview}"

            else:
                # Falls es valides JSON, aber kein Dictionary ist (z.B. eine Liste)
                result_output = f"Gepastes JSON (kein Dictionary):\n{json.dumps(data, indent=2, ensure_ascii=False)}"

        except json.JSONDecodeError:
            # 3. Wenn Parsen fehlschlägt, den String als Rohtext behandeln
            result_output = f"{result}"

        content += f"\nResult:\n{result_output}"

    else:
        # Fall, wenn der Task noch läuft
        content += "\nResult: In progress..."

    # Den gesamten Inhalt an den Formatter übergeben
    self.formatter.print_section("Tool Call", content)
print_warning(message)

Print a warning message with yellow style

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
545
546
547
548
def print_warning(self, message: str):
    """Print a warning message with yellow style"""
    if self.verbose:
        self.print(self.formatter.style.YELLOW(f"⚠️  WARNING: {message}"))
process(message, coroutine) async

Process with optional spinner

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
435
436
437
438
439
440
441
442
443
async def process(self, message: str, coroutine):
    """Process with optional spinner"""
    if not self.verbose:
        return await coroutine

    if message.lower() in ["code", "silent"]:
        return await coroutine

    return await self.formatter.process_with_spinner(message, coroutine)
section_context(title)

Context manager for sections

Source code in toolboxv2/mods/isaa/extras/verbose_output.py
526
527
528
529
530
531
532
533
534
535
@contextmanager
def section_context(self, title: str):
    """Context manager for sections"""
    if self.verbose:
        self.formatter.print_section(title, "Starting...")
    try:
        yield
    finally:
        if self.verbose:
            self.formatter.print_success(f"Completed: {title}")
clean_markdown_robust(content)

Robust markdown cleaning

Source code in toolboxv2/mods/isaa/extras/web_search.py
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
def clean_markdown_robust(content: str) -> str:
    """Robust markdown cleaning"""
    if not content:
        return ""

    # Remove common encoding artifacts more aggressively
    replacements = {
        '�': '',
        '’': "'", '“': '"', 'â€': '"', '…': '...',
        'â€"': '-', 'â€"': '--', 'Â': ' ',
        'á': 'á', 'é': 'é', 'í': 'í', 'ó': 'ó', 'ú': 'ú',
        '•': '•', '·': '·', '«': '«', '»': '»'
    }

    for old, new in replacements.items():
        content = content.replace(old, new)

    # Remove lines with too many non-ASCII characters
    lines = content.split('\n')
    cleaned_lines = []

    for line in lines:
        line = line.strip()
        if not line:
            cleaned_lines.append('')
            continue

        # Skip lines that are mostly garbled
        ascii_chars = sum(1 for c in line if ord(c) < 128)
        if len(line) > 10 and ascii_chars / len(line) < 0.7:
            continue

        # Skip navigation/junk lines
        if (len(line) < 3 or
            line.lower() in ['home', 'menu', 'search', 'login', 'register'] or
            re.match(r'^[\W\s]*$', line)):
            continue

        cleaned_lines.append(line)

    # Remove excessive empty lines
    result = '\n'.join(cleaned_lines)
    result = re.sub(r'\n{3,}', '\n\n', result)

    return result.strip()
convert_to_markdown(element)

Convert HTML element to markdown with fallbacks

Source code in toolboxv2/mods/isaa/extras/web_search.py
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
def convert_to_markdown(element):
    """Convert HTML element to markdown with fallbacks"""

    # Strategy 1: Use html2text
    try:
        import html2text
        h = html2text.HTML2Text()
        h.ignore_links = False
        h.ignore_images = True
        h.body_width = 0
        h.unicode_snob = True
        h.skip_internal_links = True
        h.inline_links = False
        h.decode_errors = 'ignore'

        markdown = h.handle(str(element))
        if markdown and len(markdown.strip()) > 100:
            return markdown
    except ImportError:
        print("html2text not installed")
    except:
        pass

    # Strategy 2: Extract text with basic formatting
    try:
        text_parts = []

        for elem in element.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
            level = int(elem.name[1])
            text_parts.append('#' * level + ' ' + elem.get_text(strip=True))
            elem.replace_with('[HEADING_PLACEHOLDER]')

        for elem in element.find_all('p'):
            text = elem.get_text(strip=True)
            if text:
                text_parts.append(text)
            elem.replace_with('[PARAGRAPH_PLACEHOLDER]')

        # Get remaining text
        remaining_text = element.get_text(separator='\n', strip=True)

        # Combine all text
        all_text = '\n\n'.join(text_parts)
        if remaining_text:
            all_text += '\n\n' + remaining_text

        return all_text

    except:
        pass

    # Strategy 3: Simple text extraction
    return element.get_text(separator='\n', strip=True)
find_main_content(soup)

Find main content using multiple strategies

Source code in toolboxv2/mods/isaa/extras/web_search.py
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
def find_main_content(soup):
    """Find main content using multiple strategies"""

    # Strategy 1: Look for semantic HTML5 elements
    for tag in ['main', 'article']:
        element = soup.find(tag)
        if element and len(element.get_text(strip=True)) > 300:
            return element

    # Strategy 2: Look for common content containers
    content_selectors = [
        '[role="main"]', '.main-content', '#main-content', '.content', '#content',
        '.post-content', '.entry-content', '.article-content', '.blog-content',
        '.story-body', '.article-body', '.post-body'
    ]

    for selector in content_selectors:
        element = soup.select_one(selector)
        if element and len(element.get_text(strip=True)) > 300:
            return element

    # Strategy 3: Find the div with most text content
    divs = soup.find_all('div')
    if divs:
        content_divs = [(div, len(div.get_text(strip=True))) for div in divs]
        content_divs = [(div, length) for div, length in content_divs if length > 300]

        if content_divs:
            content_divs.sort(key=lambda x: x[1], reverse=True)
            return content_divs[0][0]

    # Strategy 4: Use body as fallback
    return soup.find('body')
is_content_parseable(content)

Check if content is properly parsed and readable

Source code in toolboxv2/mods/isaa/extras/web_search.py
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
def is_content_parseable(content: str) -> bool:
    """
    Check if content is properly parsed and readable
    """
    if not content or len(content.strip()) < 50:
        return False

    # Check for too many non-ASCII characters that look like encoding errors
    total_chars = len(content)
    if total_chars == 0:
        return False

    # Count problematic characters
    problematic_chars = 0
    replacement_chars = content.count('�')

    # Check for sequences of garbled characters
    garbled_patterns = [
        r'[ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ]{5,}',
        r'[’“â€�]{3,}',
        r'[\x80-\xff]{4,}',  # High-byte sequences
        r'[^\x00-\x7F\s]{10,}'  # Too many non-ASCII chars in sequence
    ]

    for pattern in garbled_patterns:
        matches = re.findall(pattern, content)
        problematic_chars += sum(len(match) for match in matches)

    # Calculate ratios
    replacement_ratio = replacement_chars / total_chars
    problematic_ratio = problematic_chars / total_chars

    # Check for readable English content
    english_words = re.findall(r'\b[a-zA-Z]{3,}\b', content)
    english_ratio = len(' '.join(english_words)) / total_chars if english_words else 0

    # Criteria for parseable content
    is_parseable = (
        replacement_ratio < 0.05 and  # Less than 5% replacement chars
        problematic_ratio < 0.15 and  # Less than 15% garbled chars
        english_ratio > 0.3 and  # At least 30% English words
        len(english_words) > 10  # At least 10 English words
    )

    if not is_parseable:
        print("Content failed parseability check:")
        print(f"  Replacement ratio: {replacement_ratio:.1%}")
        print(f"  Problematic ratio: {problematic_ratio:.1%}")
        print(f"  English ratio: {english_ratio:.1%}")
        print(f"  English words: {len(english_words)}")

    return is_parseable
is_mostly_readable(text)

Check if text is mostly readable ASCII/common unicode

Source code in toolboxv2/mods/isaa/extras/web_search.py
320
321
322
323
324
325
326
def is_mostly_readable(text: str) -> bool:
    """Check if text is mostly readable ASCII/common unicode"""
    if not text:
        return False

    readable_chars = sum(1 for c in text if c.isprintable() or c.isspace())
    return readable_chars / len(text) > 0.8

Test the robust search functionality

Source code in toolboxv2/mods/isaa/extras/web_search.py
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
def robust_search():
    """Test the robust search functionality"""
    query = "Python web scraping best practices"
    results = web_search(query, max_results=3)

    print(f"\n{'=' * 60}")
    print(f"FINAL RESULTS FOR: '{query}'")
    print(f"{'=' * 60}")

    for i, result in enumerate(results, 1):
        print(f"\n{i}. {result['title']}")
        print(f"URL: {result['url']}")
        print(f"Content length: {len(result['content'])} characters")
        print(f"First 300 chars: {result['content'][:300]}...")

        # Show parseability stats
        content = result['content']
        ascii_ratio = sum(1 for c in content if ord(c) < 128) / len(content)
        print(f"ASCII ratio: {ascii_ratio:.1%}")
        print("-" * 80)
url_to_markdown_robust(url)

Robust URL to markdown converter with multiple encoding strategies

Source code in toolboxv2/mods/isaa/extras/web_search.py
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
def url_to_markdown_robust(url: str) -> str | None:
    """
    Robust URL to markdown converter with multiple encoding strategies
    """
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.9',
            'Accept-Charset': 'utf-8, iso-8859-1;q=0.5',
            'Connection': 'keep-alive'
        }

        response = requests.get(url, headers=headers, timeout=20, allow_redirects=True)
        response.raise_for_status()

        # Quick content type check
        content_type = response.headers.get('content-type', '').lower()
        if not any(ct in content_type for ct in ['text/html', 'text/plain', 'application/xhtml']):
            print(f"Skipping non-HTML content: {content_type}")
            return None

        # Get raw content
        raw_content = response.content

        # Strategy 1: Try response encoding first if it looks reliable
        decoded_content = None
        used_encoding = None

        response_encoding = response.encoding
        if response_encoding and response_encoding.lower() not in ['iso-8859-1', 'ascii']:
            try:
                decoded_content = response.text
                used_encoding = response_encoding
                # Quick test for encoding quality
                if '�' in decoded_content or not is_mostly_readable(decoded_content[:1000]):
                    decoded_content = None
            except:
                pass

        # Strategy 2: Detect encoding from content
        if not decoded_content:
            try:
                import chardet
                detected = chardet.detect(raw_content)
                if detected and detected.get('confidence', 0) > 0.8:
                    decoded_content = raw_content.decode(detected['encoding'])
                    used_encoding = detected['encoding']
                    if '�' in decoded_content or not is_mostly_readable(decoded_content[:1000]):
                        decoded_content = None
            except ImportError and ModuleNotFoundError:
                print("chardet not installed")
            except:
                pass

        # Strategy 3: Extract encoding from HTML meta tags
        if not decoded_content:
            try:
                # Try UTF-8 first to read meta tags
                temp_content = raw_content.decode('utf-8', errors='ignore')[:2048]
                charset_patterns = [
                    r'<meta[^>]+charset["\'\s]*=["\'\s]*([^"\'>\s]+)',
                    r'<meta[^>]+content[^>]+charset=([^"\'>\s;]+)',
                    r'<\?xml[^>]+encoding["\'\s]*=["\'\s]*([^"\'>\s]+)'
                ]

                for pattern in charset_patterns:
                    match = re.search(pattern, temp_content, re.I)
                    if match:
                        encoding = match.group(1).strip().lower()
                        try:
                            decoded_content = raw_content.decode(encoding)
                            used_encoding = encoding
                            if not ('�' in decoded_content or not is_mostly_readable(decoded_content[:1000])):
                                break
                        except:
                            pass
                        decoded_content = None
            except:
                pass

        # Strategy 4: Try common encodings
        if not decoded_content:
            common_encodings = ['utf-8', 'utf-8-sig', 'latin1', 'cp1252', 'iso-8859-1']
            for encoding in common_encodings:
                try:
                    test_content = raw_content.decode(encoding)
                    if is_mostly_readable(test_content[:1000]) and '�' not in test_content[:1000]:
                        decoded_content = test_content
                        used_encoding = encoding
                        break
                except:
                    continue

        # Strategy 5: Last resort with error handling
        if not decoded_content:
            decoded_content = raw_content.decode('utf-8', errors='replace')
            used_encoding = 'utf-8 (with errors)'

        print(f"Used encoding: {used_encoding}")

        # Parse with BeautifulSoup
        soup = BeautifulSoup(decoded_content, 'html.parser')

        # Remove all unwanted elements aggressively
        unwanted_tags = ['script', 'style', 'nav', 'header', 'footer', 'aside', 'iframe',
                         'form', 'button', 'input', 'noscript', 'meta', 'link', 'svg']
        for tag in unwanted_tags:
            for element in soup.find_all(tag):
                element.decompose()

        # Remove elements with unwanted classes/ids
        unwanted_patterns = [
            r'.*ad[s]?[-_].*', r'.*banner.*', r'.*popup.*', r'.*modal.*',
            r'.*cookie.*', r'.*newsletter.*', r'.*social.*', r'.*share.*',
            r'.*comment.*', r'.*sidebar.*', r'.*menu.*', r'.*navigation.*'
        ]

        for pattern in unwanted_patterns:
            for attr in ['class', 'id']:
                for element in soup.find_all(attrs={attr: re.compile(pattern, re.I)}):
                    element.decompose()

        # Find main content with multiple strategies
        main_content = find_main_content(soup)

        if not main_content:
            print("No main content found")
            return None

        # Convert to markdown using multiple strategies
        markdown_content = convert_to_markdown(main_content)

        if not markdown_content:
            print("Markdown conversion failed")
            return None

        # Clean and validate
        cleaned_content = clean_markdown_robust(markdown_content)

        # Final validation
        if not is_content_parseable(cleaned_content):
            print("Content failed parseability check")
            return None

        return cleaned_content

    except Exception as e:
        print(f"Error processing {url}: {e}")
        return None

Main search function with robust fallbacks

Source code in toolboxv2/mods/isaa/extras/web_search.py
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
def web_search(query: str, max_results: int = 5) -> list[dict[str, str]]:
    """
    Main search function with robust fallbacks
    """
    # Try API searches first if available
    api_keys = {
        'serpapi': os.getenv('SERPAPI_API_KEY'),
        'bing': os.getenv('BING_API_KEY')
    }
    if isinstance(max_results, str):
        if max_results.startswith('"') and max_results.endswith('"') or max_results.startswith("'") and max_results.endswith("'"):
            max_results = max_results[1:-1]
        max_results = int(max_results.strip())
    if api_keys:
        for api_name, api_key in api_keys.items():
            if api_key:
                try:
                    print(f"Trying {api_name.upper()} API...")
                    if api_name == 'serpapi':
                        results = web_search_serpapi(query, max_results, api_key)
                    elif api_name == 'bing':
                        results = web_search_bing(query, max_results, api_key)
                    else:
                        continue

                    if results and len(results) >= max_results:
                        return results
                except Exception as e:
                    print(f"{api_name.upper()} API failed: {e}")

    # Use robust DuckDuckGo search
    return web_search_robust(query, max_results)
web_search_bing(query, max_results=5, api_key=None)

Web search using Bing Search API (free tier: 3,000 queries/month) Get your free API key at: https://azure.microsoft.com/en-us/services/cognitive-services/bing-web-search-api/

Source code in toolboxv2/mods/isaa/extras/web_search.py
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
def web_search_bing(query: str, max_results: int = 5, api_key: str = None) -> list[dict[str, str]]:
    """
    Web search using Bing Search API (free tier: 3,000 queries/month)
    Get your free API key at: https://azure.microsoft.com/en-us/services/cognitive-services/bing-web-search-api/
    """
    if not api_key:
        print("Please get a free API key from Azure Cognitive Services")
        return []

    try:
        url = "https://api.bing.microsoft.com/v7.0/search"
        headers = {
            "Ocp-Apim-Subscription-Key": api_key
        }
        params = {
            "q": query,
            "count": max_results,
            "textDecorations": False,
            "textFormat": "HTML"
        }

        response = requests.get(url, headers=headers, params=params)
        response.raise_for_status()
        data = response.json()

        results = []
        if "webPages" in data and "value" in data["webPages"]:
            for result in data["webPages"]["value"][:max_results]:
                url_link = result.get("url", "")
                title = result.get("name", "")

                print(f"Processing: {title}")
                markdown_content = url_to_markdown_robust(url_link)

                if markdown_content:
                    results.append({
                        'url': url_link,
                        'title': title,
                        'content': markdown_content
                    })

                # time.sleep(1)

        return results

    except Exception as e:
        print(f"Bing search error: {e}")
        return []
web_search_robust(query, max_results=5, max_attempts=15)

Robust search that keeps trying until it gets enough good results

Source code in toolboxv2/mods/isaa/extras/web_search.py
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
def web_search_robust(query: str, max_results: int = 5, max_attempts: int = 15) -> list[dict[str, str]]:
    """
    Robust search that keeps trying until it gets enough good results
    """
    if isinstance(max_results, str):
        if max_results.startswith('"') and max_results.endswith('"') or max_results.startswith("'") and max_results.endswith("'"):
            max_results = max_results[1:-1]
        max_results = int(max_results.strip())
    if isinstance(max_attempts, str):
        if max_attempts.startswith('"') and max_attempts.endswith('"') or max_attempts.startswith("'") and max_attempts.endswith("'"):
            max_attempts = max_attempts[1:-1]
        max_attempts = int(max_attempts.strip())

    def get_more_search_urls(search_query: str, num_urls: int = 15) -> list[dict[str, str]]:
        """Get more URLs than needed so we can filter out bad ones"""
        try:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
                'Accept': 'text/html,application/xhtml+xml',
                'Accept-Language': 'en-US,en;q=0.9',
            }

            # Try DuckDuckGo lite
            search_url = "https://lite.duckduckgo.com/lite/"
            data = {'q': search_query}

            response = requests.post(search_url, data=data, headers=headers, timeout=15)
            response.raise_for_status()

            soup = BeautifulSoup(response.content, 'html.parser')
            results = []

            for link in soup.find_all('a', href=True):
                href = link.get('href', '')
                text = link.get_text(strip=True)

                if (href.startswith('http') and
                    'duckduckgo.com' not in href and
                    len(text) > 5 and
                    not any(skip in href.lower() for skip in ['ads', 'shopping', 'images'])):

                    results.append({
                        'url': href,
                        'title': text[:150]
                    })

                    if len(results) >= num_urls:
                        break

            return results

        except Exception as e:
            print(f"Search error: {e}")
            return []

    def get_fallback_urls(search_query: str) -> list[dict[str, str]]:
        """Get fallback URLs from known good sites"""
        encoded_query = quote_plus(search_query)
        fallback_urls = [
            f"https://stackoverflow.com/search?q={encoded_query}",
            f"https://www.reddit.com/search/?q={encoded_query}",
            f"https://medium.com/search?q={encoded_query}",
            f"https://dev.to/search?q={encoded_query}",
            f"https://github.com/search?q={encoded_query}&type=repositories",
            f"https://docs.python.org/3/search.html?q={encoded_query}",
            f"https://realpython.com/?s={encoded_query}",
            f"https://towardsdatascience.com/search?q={encoded_query}",
            f"https://www.geeksforgeeks.org/?s={encoded_query}",
            f"https://hackernoon.com/search?query={encoded_query}"
        ]

        return [
            {'url': url, 'title': f"Search results for '{search_query}'"}
            for url in fallback_urls
        ]

    print(f"Searching for: '{query}' (need {max_results} good results)")

    # Get candidate URLs
    candidate_urls = get_more_search_urls(query, max_attempts)

    if not candidate_urls:
        print("Primary search failed, using fallback URLs...")
        candidate_urls = get_fallback_urls(query)

    print(f"Found {len(candidate_urls)} candidate URLs")

    # Process URLs until we have enough good results
    good_results = []
    processed_count = 0

    def task(candidate):
        markdown_content = url_to_markdown_robust(candidate['url'])
        if markdown_content:
            return {
                'url': candidate['url'],
                'title': candidate['title'],
                'content': markdown_content
            }

    # runn all tasks in parallel
    with concurrent.futures.ThreadPoolExecutor() as executor:
        results = list(executor.map(task, candidate_urls))
        processed_count = len(candidate_urls)

    good_results = [result for result in results if result]

    #for candidate in candidate_urls:
    #    if len(good_results) >= max_results:
    #        break

    #    processed_count += 1
    #    print(f"\n[{processed_count}/{len(candidate_urls)}] Processing: {candidate['title'][:80]}...")

    #    markdown_content = url_to_markdown_robust(candidate['url'])

    #    if markdown_content:
    #        good_results.append({
    #            'url': candidate['url'],
    #            'title': candidate['title'],
    #            'content': markdown_content
    #        })
    #        print(f"✅ Success! Got result {len(good_results)}/{max_results}")
    #    else:
    #        print("❌ Skipped (unparseable or low quality)")

    #    # Small delay to be respectful
    #    time.sleep(1.5)

    print(f"\n🎉 Final results: {len(good_results)} good results out of {processed_count} attempted")
    return good_results
web_search_serpapi(query, max_results=5, api_key=None)

Web search using SerpAPI (free tier: 100 searches/month) Get your free API key at: https://serpapi.com/

Source code in toolboxv2/mods/isaa/extras/web_search.py
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
def web_search_serpapi(query: str, max_results: int = 5, api_key: str = None) -> list[dict[str, str]]:
    """
    Web search using SerpAPI (free tier: 100 searches/month)
    Get your free API key at: https://serpapi.com/
    """
    if not api_key:
        print("Please get a free API key from https://serpapi.com/")
        return []

    try:
        url = "https://serpapi.com/search"
        params = {
            "engine": "google",
            "q": query,
            "api_key": api_key,
            "num": max_results
        }

        response = requests.get(url, params=params)
        response.raise_for_status()
        data = response.json()

        results = []
        if "organic_results" in data:
            for result in data["organic_results"][:max_results]:
                url_link = result.get("link", "")
                title = result.get("title", "")

                print(f"Processing: {title}")
                markdown_content = url_to_markdown_robust(url_link)

                if markdown_content:
                    results.append({
                        'url': url_link,
                        'title': title,
                        'content': markdown_content
                    })

                #time.sleep(1)  # Be respectful

        return results

    except Exception as e:
        print(f"SerpAPI search error: {e}")
        return []

module

EnhancedAgentRequestHandler

Bases: BaseHTTPRequestHandler

Enhanced HTTP request handler for standalone server with comprehensive UI support.

Source code in toolboxv2/mods/isaa/module.py
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
class EnhancedAgentRequestHandler(BaseHTTPRequestHandler):
    """Enhanced HTTP request handler for standalone server with comprehensive UI support."""

    def __init__(self, isaa_mod, agent_id: str, agent, *args, **kwargs):
        self.isaa_mod = isaa_mod
        self.agent_id = agent_id
        self.agent = agent
        super().__init__(*args, **kwargs)

    def do_GET(self):
        """Handle GET requests for enhanced UI and status."""
        parsed_path = urlparse(self.path)

        if parsed_path.path in ['/', '/ui']:
            self._serve_enhanced_ui()
        elif parsed_path.path in ['/api/status', '/api/agent_ui/status', '/status']:
            self._serve_status()
        else:
            self._send_404()

    def do_POST(self):
        """Handle POST requests for enhanced API endpoints."""
        parsed_path = urlparse(self.path)

        if parsed_path.path in ['/api/run', '/api/agent_ui/run_agent']:
            self._handle_run_request()
        elif parsed_path.path in ['/api/reset', '/api/agent_ui/reset_context']:
            self._handle_reset_request()
        else:
            self._send_404()

    def _serve_enhanced_ui(self):
        """Serve the enhanced UI HTML."""
        try:
            html_content = get_agent_ui_html()

            self.send_response(200)
            self.send_header('Content-type', 'text/html')
            self.send_header('Content-Length', str(len(html_content.encode('utf-8'))))
            self.end_headers()
            self.wfile.write(html_content.encode('utf-8'))

        except Exception as e:
            self._send_error_response(500, f"Error serving UI: {str(e)}")

    def _serve_status(self):
        """Serve enhanced status information."""
        try:
            status_info = {
                'agent_id': self.agent_id,
                'agent_name': getattr(self.agent, 'name', 'Unknown'),
                'agent_type': self.agent.__class__.__name__,
                'status': 'active',
                'server_type': 'standalone',
                'timestamp': time.time()
            }

            if hasattr(self.agent, 'status'):
                try:
                    agent_status = self.agent.status()
                    if isinstance(agent_status, dict):
                        status_info['agent_status'] = agent_status
                except:
                    pass

            response_data = json.dumps(status_info).encode('utf-8')

            self.send_response(200)
            self.send_header('Content-type', 'application/json')
            self.send_header('Content-Length', str(len(response_data)))
            self.end_headers()
            self.wfile.write(response_data)

        except Exception as e:
            self._send_error_response(500, f"Error getting status: {str(e)}")

    def _handle_run_request(self):
        """Handle enhanced run requests with comprehensive progress tracking."""
        try:
            content_length = int(self.headers['Content-Length'])
            request_body = self.rfile.read(content_length)
            request_data = json.loads(request_body.decode('utf-8'))

            query = request_data.get('query', '')
            session_id = request_data.get('session_id', f'standalone_{secrets.token_hex(8)}')
            include_progress = request_data.get('include_progress', False)

            if not query:
                self._send_error_response(400, "Missing 'query' field")
                return

            # Run agent with enhanced progress tracking
            loop = asyncio.new_event_loop()
            asyncio.set_event_loop(loop)

            try:
                progress_tracker = EnhancedProgressTracker()
                progress_events = []
                enhanced_progress = {}

                async def standalone_progress_callback(event: ProgressEvent):
                    if include_progress:
                        progress_data = progress_tracker.extract_progress_data(event)
                        progress_events.append({
                            'timestamp': event.timestamp,
                            'event_type': event.event_type,
                            'status': getattr(event, 'status', 'unknown').value if hasattr(event, 'status') and event.status else 'unknown',
                            'data': event.to_dict()
                        })
                        enhanced_progress.update(progress_data)

                # Set progress callback
                original_callback = getattr(self.agent, 'progress_callback', None)

                if hasattr(self.agent, 'set_progress_callback'):
                    self.agent.set_progress_callback(standalone_progress_callback)
                elif hasattr(self.agent, 'progress_callback'):
                    self.agent.progress_callback = standalone_progress_callback

                # Execute agent
                result = loop.run_until_complete(
                    self.agent.a_run(query=query, session_id=session_id)
                )

                # Restore callback
                if hasattr(self.agent, 'set_progress_callback'):
                    self.agent.set_progress_callback(original_callback)
                elif hasattr(self.agent, 'progress_callback'):
                    self.agent.progress_callback = original_callback

                # Create enhanced response
                response_data = {
                    'success': True,
                    'result': result,
                    'session_id': session_id,
                    'agent_id': self.agent_id,
                    'server_type': 'standalone',
                    'timestamp': time.time()
                }

                if include_progress:
                    response_data.update({
                        'progress_events': progress_events,
                        'enhanced_progress': enhanced_progress,
                        'final_summary': progress_tracker.get_final_summary()
                    })
                self._send_json_response(response_data)

            finally:
                loop.close()

        except Exception as e:
            self._send_error_response(500, f"Execution error: {str(e)}")
            import traceback
            print(traceback.format_exc())

    def _handle_reset_request(self):
        """Handle enhanced reset requests."""
        try:
            success = False
            message = "Reset not supported"

            if hasattr(self.agent, 'clear_context'):
                self.agent.clear_context()
                success = True
                message = "Context reset successfully"
            elif hasattr(self.agent, 'reset'):
                self.agent.reset()
                success = True
                message = "Agent reset successfully"

            response_data = {
                'success': success,
                'message': message,
                'agent_id': self.agent_id,
                'timestamp': time.time()
            }

            self._send_json_response(response_data)

        except Exception as e:
            self._send_error_response(500, f"Reset error: {str(e)}")

    def _send_json_response(self, data: dict):
        """Send JSON response with CORS headers."""
        response_body = json.dumps(data, cls=CustomJSONEncoder).encode('utf-8')

        self.send_response(200)
        self.send_header('Content-type', 'application/json')
        self.send_header('Content-Length', str(len(response_body)))
        self.send_header('Access-Control-Allow-Origin', '*')
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type')
        self.end_headers()
        self.wfile.write(response_body)

    def _send_error_response(self, code: int, message: str):
        """Send error response."""
        error_data = {'success': False, 'error': message, 'code': code}
        response_body = json.dumps(error_data).encode('utf-8')

        self.send_response(code)
        self.send_header('Content-type', 'application/json')
        self.send_header('Content-Length', str(len(response_body)))
        self.end_headers()
        self.wfile.write(response_body)

    def _send_404(self):
        """Send 404 response."""
        self._send_error_response(404, "Not Found")

    def log_message(self, format, *args):
        """Override to reduce logging noise."""
        pass

    def do_OPTIONS(self):
        """Handle preflight CORS requests."""
        self.send_response(200)
        self.send_header('Access-Control-Allow-Origin', '*')
        self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
        self.send_header('Access-Control-Allow-Headers', 'Content-Type')
        self.end_headers()
do_GET()

Handle GET requests for enhanced UI and status.

Source code in toolboxv2/mods/isaa/module.py
104
105
106
107
108
109
110
111
112
113
def do_GET(self):
    """Handle GET requests for enhanced UI and status."""
    parsed_path = urlparse(self.path)

    if parsed_path.path in ['/', '/ui']:
        self._serve_enhanced_ui()
    elif parsed_path.path in ['/api/status', '/api/agent_ui/status', '/status']:
        self._serve_status()
    else:
        self._send_404()
do_OPTIONS()

Handle preflight CORS requests.

Source code in toolboxv2/mods/isaa/module.py
310
311
312
313
314
315
316
def do_OPTIONS(self):
    """Handle preflight CORS requests."""
    self.send_response(200)
    self.send_header('Access-Control-Allow-Origin', '*')
    self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS')
    self.send_header('Access-Control-Allow-Headers', 'Content-Type')
    self.end_headers()
do_POST()

Handle POST requests for enhanced API endpoints.

Source code in toolboxv2/mods/isaa/module.py
115
116
117
118
119
120
121
122
123
124
def do_POST(self):
    """Handle POST requests for enhanced API endpoints."""
    parsed_path = urlparse(self.path)

    if parsed_path.path in ['/api/run', '/api/agent_ui/run_agent']:
        self._handle_run_request()
    elif parsed_path.path in ['/api/reset', '/api/agent_ui/reset_context']:
        self._handle_reset_request()
    else:
        self._send_404()
log_message(format, *args)

Override to reduce logging noise.

Source code in toolboxv2/mods/isaa/module.py
306
307
308
def log_message(self, format, *args):
    """Override to reduce logging noise."""
    pass
EnhancedProgressTracker

Enhanced progress tracker for detailed UI updates.

Source code in toolboxv2/mods/isaa/module.py
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
class EnhancedProgressTracker:
    """Enhanced progress tracker for detailed UI updates."""

    def __init__(self):
        self.session_state = {}
        self.last_outline_update = None
        self.last_activity_update = None

    def extract_progress_data(self, event: ProgressEvent) -> dict[str, Any]:
        """Extract comprehensive progress data from event."""
        progress_data = {}

        # Outline progress
        if hasattr(event, 'outline_data') or 'outline' in event.metadata:
            outline_info = getattr(event, 'outline_data', event.metadata.get('outline', {}))
            progress_data['outline'] = {
                'current_step': outline_info.get('current_step', 'Unknown'),
                'total_steps': outline_info.get('total_steps', 0),
                'step_name': outline_info.get('step_name', 'Processing'),
                'progress_percentage': outline_info.get('progress_percentage', 0),
                'substeps': outline_info.get('substeps', []),
                'estimated_completion': outline_info.get('estimated_completion')
            }

        # Activity information
        if hasattr(event, 'activity_data') or 'activity' in event.metadata:
            activity_info = getattr(event, 'activity_data', event.metadata.get('activity', {}))
            progress_data['activity'] = {
                'current_action': activity_info.get('current_action', 'Processing'),
                'action_details': activity_info.get('action_details', ''),
                'start_time': activity_info.get('start_time'),
                'elapsed_time': activity_info.get('elapsed_time'),
                'expected_duration': activity_info.get('expected_duration')
            }

        # Meta tool information
        if hasattr(event, 'meta_tool_data') or 'meta_tool' in event.metadata:
            meta_tool_info = getattr(event, 'meta_tool_data', event.metadata.get('meta_tool', {}))
            progress_data['meta_tool'] = {
                'tool_name': meta_tool_info.get('tool_name', 'Unknown'),
                'tool_status': meta_tool_info.get('tool_status', 'active'),
                'tool_input': meta_tool_info.get('tool_input', ''),
                'tool_output': meta_tool_info.get('tool_output', ''),
                'execution_time': meta_tool_info.get('execution_time')
            }

        # System status
        if hasattr(event, 'system_data') or 'system' in event.metadata:
            system_info = getattr(event, 'system_data', event.metadata.get('system', {}))
            progress_data['system'] = {
                'memory_usage': system_info.get('memory_usage', 0),
                'cpu_usage': system_info.get('cpu_usage', 0),
                'active_threads': system_info.get('active_threads', 1),
                'queue_size': system_info.get('queue_size', 0)
            }

        # Graph/workflow information
        if hasattr(event, 'graph_data') or 'graph' in event.metadata:
            graph_info = getattr(event, 'graph_data', event.metadata.get('graph', {}))
            progress_data['graph'] = {
                'current_node': graph_info.get('current_node', 'Unknown'),
                'completed_nodes': graph_info.get('completed_nodes', []),
                'remaining_nodes': graph_info.get('remaining_nodes', []),
                'node_connections': graph_info.get('node_connections', []),
                'execution_path': graph_info.get('execution_path', [])
            }

        return progress_data
extract_progress_data(event)

Extract comprehensive progress data from event.

Source code in toolboxv2/mods/isaa/module.py
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
def extract_progress_data(self, event: ProgressEvent) -> dict[str, Any]:
    """Extract comprehensive progress data from event."""
    progress_data = {}

    # Outline progress
    if hasattr(event, 'outline_data') or 'outline' in event.metadata:
        outline_info = getattr(event, 'outline_data', event.metadata.get('outline', {}))
        progress_data['outline'] = {
            'current_step': outline_info.get('current_step', 'Unknown'),
            'total_steps': outline_info.get('total_steps', 0),
            'step_name': outline_info.get('step_name', 'Processing'),
            'progress_percentage': outline_info.get('progress_percentage', 0),
            'substeps': outline_info.get('substeps', []),
            'estimated_completion': outline_info.get('estimated_completion')
        }

    # Activity information
    if hasattr(event, 'activity_data') or 'activity' in event.metadata:
        activity_info = getattr(event, 'activity_data', event.metadata.get('activity', {}))
        progress_data['activity'] = {
            'current_action': activity_info.get('current_action', 'Processing'),
            'action_details': activity_info.get('action_details', ''),
            'start_time': activity_info.get('start_time'),
            'elapsed_time': activity_info.get('elapsed_time'),
            'expected_duration': activity_info.get('expected_duration')
        }

    # Meta tool information
    if hasattr(event, 'meta_tool_data') or 'meta_tool' in event.metadata:
        meta_tool_info = getattr(event, 'meta_tool_data', event.metadata.get('meta_tool', {}))
        progress_data['meta_tool'] = {
            'tool_name': meta_tool_info.get('tool_name', 'Unknown'),
            'tool_status': meta_tool_info.get('tool_status', 'active'),
            'tool_input': meta_tool_info.get('tool_input', ''),
            'tool_output': meta_tool_info.get('tool_output', ''),
            'execution_time': meta_tool_info.get('execution_time')
        }

    # System status
    if hasattr(event, 'system_data') or 'system' in event.metadata:
        system_info = getattr(event, 'system_data', event.metadata.get('system', {}))
        progress_data['system'] = {
            'memory_usage': system_info.get('memory_usage', 0),
            'cpu_usage': system_info.get('cpu_usage', 0),
            'active_threads': system_info.get('active_threads', 1),
            'queue_size': system_info.get('queue_size', 0)
        }

    # Graph/workflow information
    if hasattr(event, 'graph_data') or 'graph' in event.metadata:
        graph_info = getattr(event, 'graph_data', event.metadata.get('graph', {}))
        progress_data['graph'] = {
            'current_node': graph_info.get('current_node', 'Unknown'),
            'completed_nodes': graph_info.get('completed_nodes', []),
            'remaining_nodes': graph_info.get('remaining_nodes', []),
            'node_connections': graph_info.get('node_connections', []),
            'execution_path': graph_info.get('execution_path', [])
        }

    return progress_data
Tools

Bases: MainTool, FileHandler

Source code in toolboxv2/mods/isaa/module.py
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
class Tools(MainTool, FileHandler):

    def __init__(self, app=None):

        self.run_callback = None
        # self.coding_projects: dict[str, ProjectManager] = {} # Assuming ProjectManager is defined elsewhere or removed
        if app is None:
            app = get_app("isaa-mod")
        self.version = version
        self.name = "isaa"
        self.Name = "isaa"
        self.color = "VIOLET2"
        self.config = {'controller-init': False,
                       'agents-name-list': [], # TODO Remain ComplexModel FastModel BlitzModel, AudioModel, (ImageModel[i/o], VideoModel[i/o]), SummaryModel
                       "FASTMODEL": os.getenv("FASTMODEL", "ollama/llama3.1"),
                       "AUDIOMODEL": os.getenv("AUDIOMODEL", "groq/whisper-large-v3-turbo"),
                       "BLITZMODEL": os.getenv("BLITZMODEL", "ollama/llama3.1"),
                       "COMPLEXMODEL": os.getenv("COMPLEXMODEL", "ollama/llama3.1"),
                       "SUMMARYMODEL": os.getenv("SUMMARYMODEL", "ollama/llama3.1"),
                       "IMAGEMODEL": os.getenv("IMAGEMODEL", "ollama/llama3.1"),
                       "DEFAULTMODELEMBEDDING": os.getenv("DEFAULTMODELEMBEDDING", "gemini/text-embedding-004"),
                       }
        self.per_data = {}
        self.agent_data: dict[str, dict] = {}  # Will store AgentConfig dicts
        self.keys = {
            "KEY": "key~~~~~~~",
            "Config": "config~~~~"
        }
        self.initstate = {}

        extra_path = ""
        if self.toolID:  # MainTool attribute
            extra_path = f"/{self.toolID}"
        self.observation_term_mem_file = f"{app.data_dir}/Memory{extra_path}/observationMemory/"
        self.config['controller_file'] = f"{app.data_dir}{extra_path}/controller.json"
        self.mas_text_summaries_dict = FileCache(folder=f"{app.data_dir}/Memory{extra_path}/summaries/")
        self.tools = {
            "name": "isaa",
            "Version": self.show_version,
            "mini_task_completion": self.mini_task_completion,
            "run_agent": self.run_agent,
            "save_to_mem": self.save_to_mem_sync,
            "get_agent": self.get_agent,
            "format_class": self.format_class,  # Now async
            "get_memory": self.get_memory,
            "save_all_memory_vis": self.save_all_memory_vis,
            "rget_mode": lambda mode: self.controller.rget(mode),
        }
        self.tools_interfaces: dict[str, ToolsInterface] = {}
        self.working_directory = os.getenv('ISAA_WORKING_PATH', os.getcwd())
        self.print_stream = stram_print
        self.global_stream_override = False  # Handled by FlowAgentBuilder
        self.lang_chain_tools_dict: dict[str, Any] = {}  # Store actual tool objects for wrapping

        self.agent_memory: AISemanticMemory = f"{app.id}{extra_path}/Memory"  # Path for AISemanticMemory
        self.controller = ControllerManager({})
        self.summarization_mode = 1
        self.summarization_limiter = 102000
        self.speak = lambda x, *args, **kwargs: x  # Placeholder

        self.default_setter = None  # For agent builder customization
        self.initialized = False

        FileHandler.__init__(self, f"isaa{extra_path.replace('/', '-')}.config", app.id if app else __name__)
        MainTool.__init__(self, load=self.on_start, v=self.version, tool=self.tools,
                          name=self.name, logs=None, color=self.color, on_exit=self.on_exit)

        from .extras.web_search import web_search
        async def web_search_tool(query: str) -> str:
            res = web_search(query)
            return await self.mas_text_summaries(str(res), min_length=12000, ref=query)
        self.web_search = web_search_tool
        self.shell_tool_function = shell_tool_function
        self.tools["shell"] = shell_tool_function

        self.print(f"Start {self.spec}.isaa")
        with Spinner(message="Starting module", symbols='c'):
            self.load_file_handler()
            config_fh = self.get_file_handler(self.keys["Config"])
            if config_fh is not None:
                if isinstance(config_fh, str):
                    try:
                        config_fh = json.loads(config_fh)
                    except json.JSONDecodeError:
                        self.print(f"Warning: Could not parse config from file handler: {config_fh[:100]}...")
                        config_fh = {}

                if isinstance(config_fh, dict):
                    # Merge, prioritizing existing self.config for defaults not in file
                    loaded_config = config_fh
                    for key, value in self.config.items():
                        if key not in loaded_config:
                            loaded_config[key] = value
                    self.config = loaded_config

            if self.spec == 'app':  # MainTool attribute
                self.load_keys_from_env()

            # Ensure directories exist
            Path(f"{get_app('isaa-initIsaa').data_dir}/Agents/").mkdir(parents=True, exist_ok=True)
            Path(f"{get_app('isaa-initIsaa').data_dir}/Memory/").mkdir(parents=True, exist_ok=True)

        #initialize_isaa_webui_module(self.app, self)
        #self.print("ISAA module started. fallback")

    def get_augment(self):
        # This needs to be adapted. Serialization of FlowAgent is through AgentConfig.
        return {
            "Agents": self.serialize_all(),  # Returns dict of AgentConfig dicts
        }

    async def init_from_augment(self, augment, agent_name: str = 'self'):
        """Initialize from augmented data using new builder system"""

        # Handle agent_name parameter
        if isinstance(agent_name, str):
            pass  # Use string name
        elif hasattr(agent_name, 'config'):  # FlowAgentBuilder
            agent_name = agent_name.config.name
        else:
            raise ValueError(f"Invalid agent_name type: {type(agent_name)}")

        a_keys = augment.keys()

        # Load agent configurations
        if "Agents" in a_keys:
            agents_configs_dict = augment['Agents']
            self.deserialize_all(agents_configs_dict)
            self.print("Agent configurations loaded.")

        # Tools are now handled by the builder system during agent creation
        if "tools" in a_keys:
            self.print("Tool configurations noted - will be applied during agent building")

    async def init_tools(self, tools_config: dict, agent_builder: FlowAgentBuilder):
        # This function needs to be adapted to add tools to the FlowAgentBuilder
        # For LangChain tools, they need to be wrapped as callables or ADK BaseTool instances.
        lc_tools_names = tools_config.get('lagChinTools', [])
        # hf_tools_names = tools_config.get('huggingTools', []) # HuggingFace tools are also LangChain tools
        # plugin_urls = tools_config.get('Plugins', [])

        all_lc_tool_names = list(set(lc_tools_names))  # + hf_tools_names

        for tool_name in all_lc_tool_names:
            try:
                # Load tool instance (LangChain's load_tools might return a list)
                loaded_tools = load_tools([tool_name], llm=None)  # LLM not always needed for tool definition
                for lc_tool_instance in loaded_tools:
                    # Wrap and add to builder
                    # Simple case: wrap lc_tool_instance.run or lc_tool_instance._run
                    if hasattr(lc_tool_instance, 'run') and callable(lc_tool_instance.run):
                        # ADK FunctionTool needs a schema, or infers it.
                        # We might need to manually create Pydantic models for args.
                        # For simplicity, assume ADK can infer or the tool takes simple args.
                        agent_builder.add_tool(lc_tool_instance.run, name=lc_tool_instance.name,
                                                             description=lc_tool_instance.description)
                        self.print(f"Added LangChain tool '{lc_tool_instance.name}' to builder.")
                        self.lang_chain_tools_dict[lc_tool_instance.name] = lc_tool_instance  # Store for reference
            except Exception as e:
                self.print(f"Failed to load/add LangChain tool '{tool_name}': {e}")

        # AIPluginTool needs more complex handling as it's a class
        # for url in plugin_urls:
        #     try:
        #         plugin = AIPluginTool.from_plugin_url(url)
        #         # Exposing AIPluginTool methods might require creating individual FunctionTools
        #         # Or creating a custom ADK BaseTool wrapper for AIPluginTool
        #         self.print(f"AIPluginTool {plugin.name} loaded. Manual ADK wrapping needed.")
        #     except Exception as e:
        #         self.print(f"Failed to load AIPlugin from {url}: {e}")

    def serialize_all(self):
        # Returns a copy of agent_data, which contains AgentConfig dicts
        # The exclude logic might be different if it was excluding fields from old AgentBuilder
        # For AgentConfig, exclusion happens during model_dump if needed.
        return copy.deepcopy(self.agent_data)

    def deserialize_all(self, data: dict[str, dict]):
        # Data is a dict of {agent_name: builder_config_dict}
        self.agent_data.update(data)
        # Clear instances from self.config so they are rebuilt with new configs
        for agent_name in data:
            self.config.pop(f'agent-instance-{agent_name}', None)

    async def init_isaa(self, name='self', build=False, **kwargs):
        if self.initialized:
            self.print(f"Already initialized. Getting agent/builder: {name}")
            # build=True implies getting the builder, build=False (default) implies getting agent instance
            return self.get_agent_builder(name) if build else await self.get_agent(name)

        self.initialized = True
        sys.setrecursionlimit(1500)
        self.load_keys_from_env()

        with Spinner(message="Building Controller", symbols='c'):
            self.controller.init(self.config['controller_file'])
        self.config["controller-init"] = True


        return self.get_agent_builder(name) if build else await self.get_agent(name)

    def show_version(self):
        self.print("Version: ", self.version)
        return self.version

    def on_start(self):

        initialize_isaa_webui_module(self.app, self)

        threading.Thread(target=self.load_to_mem_sync, daemon=True).start()
        self.print("ISAA module started.")

    def load_keys_from_env(self):
        # Update default model names from environment variables
        for key in self.config:
            if key.startswith("DEFAULTMODEL"):
                self.config[key] = os.getenv(key, self.config[key])
        self.config['VAULTS'] = os.getenv("VAULTS")

    def on_exit(self):
        self.app.run_bg_task_advanced(self.cleanup_tools_interfaces)
        # Save agent configurations
        for agent_name, agent_instance in self.config.items():
            if agent_name.startswith('agent-instance-') and agent_instance and isinstance(agent_instance, list) and isinstance(agent_instance[0], FlowAgent):
                self.app.run_bg_task_advanced(asyncio.gather(*[agent_instance.close() for agent_instance in agent_instance]))
                # If agent instance has its own save logic (e.g. cost tracker)
                # asyncio.run(agent_instance.close()) # This might block, consider task group
                # The AgentConfig is already in self.agent_data, which should be saved.
                pass  # Agent instances are not directly saved, their configs are.
        threading.Thread(target=self.save_to_mem_sync, daemon=True).start()  # Sync wrapper for save_to_mem

        # Save controller if initialized
        if self.config.get("controller-init"):
            self.controller.save(self.config['controller_file'])

        # Clean up self.config for saving
        clean_config = {}
        for key, value in self.config.items():
            if key.startswith('agent-instance-'): continue  # Don't save instances
            if key.startswith('LLM-model-'): continue  # Don't save langchain models
            clean_config[key] = value
        self.add_to_save_file_handler(self.keys["Config"], json.dumps(clean_config))

        # Save other persistent data
        self.save_file_handler()

    def save_to_mem_sync(self):
        # This used to call agent.save_memory(). FlowAgent does not have this.
        # If AISemanticMemory needs global saving, it should be handled by AISemanticMemory itself.
        # For now, this can be a no-op or save AISemanticMemory instances if managed by Tools.
        memory_instance = self.get_memory()  # Assuming this returns AISemanticMemory
        if hasattr(memory_instance, 'save_all_memories'):  # Hypothetical method
            memory_instance.save_all_memories(f"{get_app().data_dir}/Memory/")
        self.print("Memory saving process initiated")

    def load_to_mem_sync(self):
        # This used to call agent.save_memory(). FlowAgent does not have this.
        # If AISemanticMemory needs global saving, it should be handled by AISemanticMemory itself.
        # For now, this can be a no-op or save AISemanticMemory instances if managed by Tools.
        memory_instance = self.get_memory()  # Assuming this returns AISemanticMemory
        if hasattr(memory_instance, 'load_all_memories'):  # Hypothetical method
            memory_instance.load_all_memories(f"{get_app().data_dir}/Memory/")
        self.print("Memory loading process initiated")

    def get_agent_builder(self, name="self", extra_tools=None, add_tools=True, add_base_tools=True, working_directory=None) -> FlowAgentBuilder:
        if name == 'None':
            name = "self"

        if extra_tools is None:
            extra_tools = []

        self.print(f"Creating FlowAgentBuilder: {name}")

        # Create builder with agent-specific configuration
        config = AgentConfig(
            name=name,
            fast_llm_model=self.config.get(f'{name.upper()}MODEL', self.config['FASTMODEL']),
            complex_llm_model=self.config.get(f'{name.upper()}MODEL', self.config['COMPLEXMODEL']),
            system_message="You are a production-ready autonomous agent.",
            temperature=0.7,
            max_tokens_output=2048,
            max_tokens_input=32768,
            use_fast_response=True,
            max_parallel_tasks=3,
            verbose_logging=False
        )

        builder = FlowAgentBuilder(config=config)
        builder._isaa_ref = self  # Store ISAA reference

        # Load existing configuration if available
        agent_config_path = Path(f"{get_app().data_dir}/Agents/{name}/agent.json")
        if agent_config_path.exists():
            try:
                builder = FlowAgentBuilder.from_config_file(str(agent_config_path))
                builder._isaa_ref = self
                self.print(f"Loaded existing configuration for builder {name}")
            except Exception as e:
                self.print(f"Failed to load config for {name}: {e}. Using defaults.")

        # Apply global settings
        if self.global_stream_override:
            builder.verbose(True)

        # Apply custom setter if available
        if self.default_setter:
            builder = self.default_setter(builder, name)

        # Initialize ToolsInterface for this agent
        if not hasattr(self, 'tools_interfaces'):
            self.tools_interfaces = {}

        # Create or get existing ToolsInterface for this agent
        if name not in self.tools_interfaces:
            try:
                # Initialize ToolsInterface
                p = Path(get_app().data_dir) / "Agents" / name / "tools_session"
                p.mkdir(parents=True, exist_ok=True)
                tools_interface = ToolsInterface(
                    session_dir=str(Path(get_app().data_dir) / "Agents" / name / "tools_session"),
                    auto_remove=False,  # Keep session data for agents
                    variables={
                        'agent_name': name,
                        'isaa_instance': self
                    },
                    variable_manager=getattr(self, 'variable_manager', None),
                )
                if working_directory:
                    tools_interface.set_base_directory(working_directory)

                self.tools_interfaces[name] = tools_interface
                self.print(f"Created ToolsInterface for agent: {name}")

            except Exception as e:
                self.print(f"Failed to create ToolsInterface for {name}: {e}")
                self.tools_interfaces[name] = None

        tools_interface = self.tools_interfaces[name]

        # Add ISAA core tools
        async def run_isaa_agent_tool(target_agent_name: str, instructions: str, **kwargs_):
            if not instructions:
                return "No instructions provided."
            if target_agent_name.startswith('"') and target_agent_name.endswith('"') or target_agent_name.startswith(
                "'") and target_agent_name.endswith("'"):
                target_agent_name = target_agent_name[1:-1]
            return await self.run_agent(target_agent_name, text=instructions, **kwargs_)

        async def memory_search_tool(
            query: str,
            search_mode: str | None = "balanced",
            context_name: str | None = None
        ) -> str:
            """Memory search with configurable precision"""
            mem_instance = self.get_memory()
            memory_names_list = [name.strip() for name in context_name.split(',')] if context_name else None

            search_params = {
                "wide": {"k": 7, "min_similarity": 0.1, "cross_ref_depth": 3, "max_cross_refs": 4, "max_sentences": 8},
                "narrow": {"k": 2, "min_similarity": 0.75, "cross_ref_depth": 1, "max_cross_refs": 1,
                           "max_sentences": 3},
                "balanced": {"k": 3, "min_similarity": 0.2, "cross_ref_depth": 2, "max_cross_refs": 2,
                             "max_sentences": 5}
            }.get(search_mode,
                  {"k": 3, "min_similarity": 0.2, "cross_ref_depth": 2, "max_cross_refs": 2, "max_sentences": 5})

            return await mem_instance.query(
                query=query, memory_names=memory_names_list,
                query_params=search_params, to_str=True
            )

        async def save_to_memory_tool(data_to_save: str, context_name: str = name):
            mem_instance = self.get_memory()
            result = await mem_instance.add_data(context_name, str(data_to_save), direct=True)
            return 'Data added to memory.' if result else 'Error adding data to memory.'

        # Add ISAA core tools


        if add_base_tools:
            builder.add_tool(memory_search_tool, "memorySearch", "Search ISAA's semantic memory")
            builder.add_tool(save_to_memory_tool, "saveDataToMemory", "Save data to ISAA's semantic memory")
            builder.add_tool(self.web_search, "searchWeb", "Search the web for information")
            builder.add_tool(self.shell_tool_function, "shell", f"Run shell command in {detect_shell()}")

        # Add ToolsInterface tools dynamically
        if add_tools and tools_interface:
            try:
                # Get all tools from ToolsInterface
                interface_tools = tools_interface.get_tools()

                # Determine which tools to add based on agent name/type
                tool_categories = {
                    'code': ['execute_python', 'install_package'],
                    'file': ['write_file', 'replace_in_file', 'read_file', 'list_directory', 'create_directory'],
                    'session': ['get_execution_history', 'clear_session', 'get_variables'],
                    'config': ['set_base_directory', 'set_current_file']
                }

                # Determine which categories to include
                include_categories = set()
                name_lower = name.lower()

                # Code execution for development/coding agents
                if any(keyword in name_lower for keyword in ["dev", "code", "program", "script", "python", "rust", "worker"]):
                    include_categories.update(['code', 'file', 'session', 'config'])

                # Web tools for web-focused agents
                if any(keyword in name_lower for keyword in ["web", "browser", "scrape", "crawl", "extract"]):
                    include_categories.update(['file', 'session'])

                # File tools for file management agents
                if any(keyword in name_lower for keyword in ["file", "fs", "document", "write", "read"]):
                    include_categories.update(['file', 'session', 'config'])

                # Default: add core tools for general agents
                if not include_categories or name == "self":
                    include_categories.update(['code', 'file', 'session', 'config'])

                # Add selected tools
                tools_added = 0
                for tool_func, tool_name, tool_description in interface_tools:
                    # Check if this tool should be included
                    should_include = tool_name in extra_tools

                    if not should_include:
                        for category, tool_names in tool_categories.items():
                            if category in include_categories and tool_name in tool_names:
                                should_include = True
                                break

                    # Always include session management tools
                    if tool_name in ['get_execution_history', 'get_variables']:
                        should_include = True

                    if should_include:
                        try:
                            builder.add_tool(tool_func, tool_name, tool_description)
                            tools_added += 1
                        except Exception as e:
                            self.print(f"Failed to add tool {tool_name}: {e}")

                self.print(f"Added {tools_added} ToolsInterface tools to agent {name}")

            except Exception as e:
                self.print(f"Error adding ToolsInterface tools to {name}: {e}")

        # Configure cost tracking
        builder.with_budget_manager(max_cost=100.0)

        # Store agent configuration
        try:
            agent_dir = Path(f"{get_app().data_dir}/Agents/{name}")
            agent_dir.mkdir(parents=True, exist_ok=True)

            # Save agent metadata
            metadata = {
                'name': name,
                'created_at': time.time(),
                'tools_interface_available': tools_interface is not None,
                'session_dir': str(agent_dir / "tools_session")
            }

            metadata_file = agent_dir / "metadata.json"
            with open(metadata_file, 'w') as f:
                json.dump(metadata, f, indent=2)

        except Exception as e:
            self.print(f"Failed to save agent metadata for {name}: {e}")

        return builder

    def get_tools_interface(self, agent_name: str = "self") -> ToolsInterface | None:
        """
        Get the ToolsInterface instance for a specific agent.

        Args:
            agent_name: Name of the agent

        Returns:
            ToolsInterface instance or None if not found
        """
        if not hasattr(self, 'tools_interfaces'):
            return None

        return self.tools_interfaces.get(agent_name)

    async def configure_tools_interface(self, agent_name: str, **kwargs) -> bool:
        """
        Configure the ToolsInterface for a specific agent.

        Args:
            agent_name: Name of the agent
            **kwargs: Configuration parameters

        Returns:
            True if successful, False otherwise
        """
        tools_interface = self.get_tools_interface(agent_name)
        if not tools_interface:
            self.print(f"No ToolsInterface found for agent {agent_name}")
            return False

        try:
            # Configure based on provided parameters
            if 'base_directory' in kwargs:
                await tools_interface.set_base_directory(kwargs['base_directory'])

            if 'current_file' in kwargs:
                await tools_interface.set_current_file(kwargs['current_file'])

            if 'variables' in kwargs:
                tools_interface.ipython.user_ns.update(kwargs['variables'])

            self.print(f"Configured ToolsInterface for agent {agent_name}")
            return True

        except Exception as e:
            self.print(f"Failed to configure ToolsInterface for {agent_name}: {e}")
            return False

    async def cleanup_tools_interfaces(self):
        """
        Cleanup all ToolsInterface instances.
        """
        if not hasattr(self, 'tools_interfaces'):
            return

        async def cleanup_async():
            for name, tools_interface in self.tools_interfaces.items():
                if tools_interface:
                    try:
                        await tools_interface.__aexit__(None, None, None)
                    except Exception as e:
                        self.print(f"Error cleaning up ToolsInterface for {name}: {e}")

        # Run cleanup
        try:
            await cleanup_async()
            self.tools_interfaces.clear()
            self.print("Cleaned up all ToolsInterface instances")
        except Exception as e:
            self.print(f"Error during ToolsInterface cleanup: {e}")

    async def register_agent(self, agent_builder: FlowAgentBuilder):
        agent_name = agent_builder.config.name

        if f'agent-instance-{agent_name}' in self.config:
            self.print(f"Agent '{agent_name}' instance already exists. Overwriting config and rebuilding on next get.")
            self.config.pop(f'agent-instance-{agent_name}', None)

        # Save the builder's configuration
        config_path = Path(f"{get_app().data_dir}/Agents/{agent_name}/agent.json")
        agent_builder.save_config(str(config_path), format='json')
        self.print(f"Saved FlowAgentBuilder config for '{agent_name}' to {config_path}")

        # Store serializable config in agent_data
        self.agent_data[agent_name] = agent_builder.config.model_dump()

        if agent_name not in self.config.get("agents-name-list", []):
            if "agents-name-list" not in self.config:
                self.config["agents-name-list"] = []
            self.config["agents-name-list"].append(agent_name)

        self.print(f"FlowAgent '{agent_name}' configuration registered. Will be built on first use.")
        row_agent_builder_sto[agent_name] = agent_builder  # Cache builder

    async def get_agent(self, agent_name="Normal", model_override: str | None = None) -> FlowAgent:
        if "agents-name-list" not in self.config:
            self.config["agents-name-list"] = []

        instance_key = f'agent-instance-{agent_name}'
        if instance_key in self.config:
            agent_instance = self.config[instance_key]
            if model_override and agent_instance.amd.fast_llm_model != model_override:
                self.print(f"Model override for {agent_name}: {model_override}. Rebuilding.")
                self.config.pop(instance_key, None)
            else:
                self.print(f"Returning existing FlowAgent instance: {agent_name}")
                return agent_instance

        builder_to_use = None

        # Try to get cached builder first
        if agent_name in row_agent_builder_sto:
            builder_to_use = row_agent_builder_sto[agent_name]
            self.print(f"Using cached builder for {agent_name}")

        # Try to load from stored config
        elif agent_name in self.agent_data:
            self.print(f"Loading configuration for FlowAgent: {agent_name}")
            try:
                config = AgentConfig(**self.agent_data[agent_name])
                builder_to_use = FlowAgentBuilder(config=config)
            except Exception as e:
                self.print(f"Error loading config for {agent_name}: {e}. Falling back to default.")

        # Create default builder if none found
        if builder_to_use is None:
            self.print(f"No existing config for {agent_name}. Creating default builder.")
            builder_to_use = self.get_agent_builder(agent_name)

        # Apply overrides and ensure correct name
        builder_to_use._isaa_ref = self
        if model_override:
            builder_to_use.with_models(model_override, model_override)

        if builder_to_use.config.name != agent_name:
            builder_to_use.with_name(agent_name)

        self.print(
            f"Building FlowAgent: {agent_name} with models {builder_to_use.config.fast_llm_model} - {builder_to_use.config.complex_llm_model}")

        # Build the agent
        agent_instance: FlowAgent = await builder_to_use.build()

        if agent_instance.amd.name == "self":
            self.app.run_bg_task_advanced(agent_instance.initialize_context_awareness)

        if interface := self.get_tools_interface(agent_name):
            interface.variable_manager = agent_instance.variable_manager

        # colletive cabability cahring for reduched reduanda analysis _tool_capabilities
        agent_tool_nams = set(agent_instance.tool_registry.keys())

        tools_data = {}
        for _agent_name in self.config["agents-name-list"]:
            _instance_key = f'agent-instance-{_agent_name}'
            if _instance_key not in self.config:
                if agent_name != "self" and _agent_name == "self":
                    await self.get_agent("self")

            if _instance_key not in self.config:
                continue
            _agent_instance = self.config[_instance_key]
            _agent_tool_nams = set(_agent_instance._tool_capabilities.keys())
            # extract the tool names that are in both agents_registry
            overlap_tool_nams = agent_tool_nams.intersection(_agent_tool_nams)
            _tc = _agent_instance._tool_capabilities
            for tool_name in overlap_tool_nams:
                if tool_name not in _tc:
                    continue
                tools_data[tool_name] = _tc[tool_name]

        agent_instance._tool_capabilities.update(tools_data)
        # Cache the instance and update tracking
        self.config[instance_key] = agent_instance
        if agent_name not in self.agent_data:
            self.agent_data[agent_name] = builder_to_use.config.model_dump()
        if agent_name not in self.config["agents-name-list"]:
            self.config["agents-name-list"].append(agent_name)

        self.print(f"Built and cached FlowAgent instance: {agent_name}")
        return agent_instance

    @export(api=True, version=version, request_as_kwarg=True, mod_name="isaa")
    async def mini_task_completion(self, mini_task: str | None = None, user_task: str | None = None, mode: Any = None,  # LLMMode
                                   max_tokens_override: int | None = None, task_from="system",
                                   stream_function: Callable | None = None, message_history: list | None = None, agent_name="TaskCompletion", use_complex: bool = False, request: RequestData | None = None, form_data: dict | None = None, data: dict | None = None, **kwargs):
        if request is not None or form_data is not None or data is not None:
            data_dict = (request.request.body if request else None) or form_data or data
            mini_task = mini_task or  data_dict.get("mini_task")
            user_task = user_task or data_dict.get("user_task")
            mode = mode or data_dict.get("mode")
            max_tokens_override = max_tokens_override or data_dict.get("max_tokens_override")
            task_from = data_dict.get("task_from") or task_from
            agent_name = data_dict.get("agent_name") or agent_name
            use_complex = use_complex or data_dict.get("use_complex")
            kwargs = kwargs or data_dict.get("kwargs")
            message_history = message_history or data_dict.get("message_history")
            if isinstance(message_history, str):
                message_history = json.loads(message_history)
        print(mini_task, agent_name, use_complex, kwargs, message_history, form_data or data)
        if mini_task is None: return None
        if agent_name is None: return None
        if mini_task == "test": return "test"
        self.print(f"Running mini task, volume {len(mini_task)}")

        agent = await self.get_agent(agent_name)  # Ensure agent is retrieved (and built if needed)

        effective_system_message = agent.amd.system_message
        if mode and hasattr(mode, 'system_msg') and mode.system_msg:
            effective_system_message = mode.system_msg

        messages = []
        if effective_system_message:
            messages.append({"role": "system", "content": effective_system_message})
        if message_history:
            messages.extend(message_history)

        current_prompt = mini_task
        if user_task:  # If user_task is provided, it becomes the main prompt, mini_task is context
            messages.append({"role": task_from, "content": mini_task})  # mini_task as prior context
            current_prompt = user_task  # user_task as the current prompt

        messages.append({"role": "user", "content": current_prompt})

        # Prepare params for a_run_llm_completion
        if use_complex:
            llm_params = {"model": agent.amd.complex_llm_model, "messages": messages}
        else:
            llm_params = {"model": agent.amd.fast_llm_model if agent.amd.use_fast_response else agent.amd.complex_llm_model, "messages": messages}
        if max_tokens_override:
            llm_params['max_tokens'] = max_tokens_override
        else:
            llm_params['max_tokens'] = agent.amd.max_tokens
        if kwargs:
            llm_params.update(kwargs)  # Add any additional kwargs
        if stream_function:
            llm_params['stream'] = True
            # FlowAgent a_run_llm_completion handles stream_callback via agent.stream_callback
            # For a one-off, we might need a temporary override or pass it if supported.
            # For now, assume stream_callback is set on agent instance if needed globally.
            # If stream_function is for this call only, agent.a_run_llm_completion needs modification
            # or we use a temporary agent instance. This part is tricky.
            # Let's assume for now that if stream_function is passed, it's a global override for this agent type.
            original_stream_cb = agent.stream_callback
            original_stream_val = agent.stream
            agent.stream_callback = stream_function
            agent.stream = True
            try:
                response_content = await agent.a_run_llm_completion(**llm_params)
            finally:
                agent.stream_callback = original_stream_cb
                agent.stream = original_stream_val  # Reset to builder's config
            return response_content  # Streaming output handled by callback

        llm_params['stream'] = False
        response_content = await agent.a_run_llm_completion(**llm_params)
        return response_content

    async def mini_task_completion_format(self, mini_task, format_schema: type[BaseModel],
                                          max_tokens_override: int | None = None, agent_name="TaskCompletion",
                                          task_from="system", mode_overload: Any = None, user_task: str | None = None, auto_context=False, **kwargs):
        if mini_task is None: return None
        self.print(f"Running formatted mini task, volume {len(mini_task)}")

        agent = await self.get_agent(agent_name)

        effective_system_message = None
        if mode_overload and hasattr(mode_overload, 'system_msg') and mode_overload.system_msg:
            effective_system_message = mode_overload.system_msg

        message_context = []
        if effective_system_message:
            message_context.append({"role": "system", "content": effective_system_message})

        current_prompt = mini_task
        if user_task:
            message_context.append({"role": task_from, "content": mini_task})
            current_prompt = user_task

        # Use agent.a_format_class
        try:
            result_dict = await agent.a_format_class(
                pydantic_model=format_schema,
                prompt=current_prompt,
                message_context=message_context,
                auto_context=auto_context
                # max_tokens can be part of agent's model config or passed if a_format_class supports it
            )
            if format_schema == bool:  # Special handling for boolean schema
                # a_format_class returns a dict, e.g. {"value": True}. Extract the bool.
                # This depends on how bool schema is defined. A common way: class BoolResponse(BaseModel): value: bool
                return result_dict.get("value", False) if isinstance(result_dict, dict) else False
            return result_dict
        except Exception as e:
            self.print(f"Error in mini_task_completion_format: {e}")
            return None  # Or raise

    @export(api=True, version=version, name="version")
    async def get_version(self, *a,**k):
        return self.version

    @export(api=True, version=version, request_as_kwarg=True, mod_name="isaa")
    async def format_class(self, format_schema: type[BaseModel] | None = None, task: str | None = None, agent_name="TaskCompletion", auto_context=False, request: RequestData | None = None, form_data: dict | None = None, data: dict | None = None, **kwargs):
        if request is not None or form_data is not None or data is not None:
            data_dict = (request.request.body if request else None) or form_data or data
            format_schema = format_schema or data_dict.get("format_schema")
            task = task or data_dict.get("task")
            agent_name = data_dict.get("agent_name") or agent_name
            auto_context = auto_context or data_dict.get("auto_context")
            kwargs = kwargs or data_dict.get("kwargs")
        if format_schema is None or not task: return None
        agent = None
        if isinstance(agent_name, str):
            agent = await self.get_agent(agent_name)
        elif isinstance(agent_name, FlowAgent):
            agent = agent_name
        else:
            raise TypeError("agent_name must be str or FlowAgent instance")

        return await agent.a_format_class(format_schema, task, auto_context=auto_context)

    async def run_agent(self, name: str | FlowAgent,
                        text: str,
                        verbose: bool = False,  # Handled by agent's own config mostly
                        session_id: str | None = None,
                        progress_callback: Callable[[Any], None | Awaitable[None]] | None = None,
                        **kwargs):  # Other kwargs for a_run
        if text is None: return ""
        if name is None: return ""
        if text == "test": return ""

        agent_instance = None
        if isinstance(name, str):
            agent_instance = await self.get_agent(name)
        elif isinstance(name, FlowAgent):
            agent_instance = name
        else:
            return self.return_result().default_internal_error(
                f"Invalid agent identifier type: {type(name)}")

        self.print(f"Running agent {agent_instance.amd.name} for task: {text[:100]}...")
        save_p = None
        if progress_callback:
            save_p = agent_instance.progress_callback
            agent_instance.progress_callback = progress_callback

        if verbose:
            agent_instance.verbose = True

        # Call FlowAgent's a_run method
        response = await agent_instance.a_run(
            query=text,
            session_id=session_id,
            user_id=None,
            stream_callback=None

        )
        if save_p:
            agent_instance.progress_callback = save_p

        return response

    # mass_text_summaries and related methods remain complex and depend on AISemanticMemory
    # and specific summarization strategies. For now, keeping their structure,
    # but calls to self.format_class or self.mini_task_completion will become async.

    async def mas_text_summaries(self, text, min_length=36000, ref=None, max_tokens_override=None):
        len_text = len(text)
        if len_text < min_length: return text
        key = self.one_way_hash(text, 'summaries', 'isaa')
        value = self.mas_text_summaries_dict.get(key)
        if value is not None: return value

        # This part needs to become async due to format_class
        # Simplified version:
        from .extras.modes import (
            SummarizationMode,
            # crate_llm_function_from_langchain_tools,
        )
        summary = await self.mini_task_completion(
            mini_task=f"Summarize this text, focusing on aspects related to '{ref if ref else 'key details'}'. The text is: {text}",
            mode=self.controller.rget(SummarizationMode), max_tokens_override=max_tokens_override, agent_name="self")

        if summary is None or not isinstance(summary, str):
            # Fallback or error handling
            summary = text[:min_length] + "... (summarization failed)"

        self.mas_text_summaries_dict.set(key, summary)
        return summary

    def get_memory(self, name: str | None = None) -> AISemanticMemory:
        # This method's logic seems okay, AISemanticMemory is a separate system.
        logger_ = get_logger()  # Renamed to avoid conflict with self.logger
        if isinstance(self.agent_memory, str):  # Path string
            logger_.info(Style.GREYBG("AISemanticMemory Initialized from path"))
            self.agent_memory = AISemanticMemory(base_path=self.agent_memory)

        cm = self.agent_memory
        if name is not None:
            # Assuming AISemanticMemory.get is synchronous or you handle async appropriately
            # If AISemanticMemory methods become async, this needs adjustment
            mem_kb = cm.get(name)  # This might return a list of KnowledgeBase or single one
            return mem_kb
        return cm

    async def save_all_memory_vis(self, dir_path=None):
        if dir_path is None:
            dir_path = f"{get_app().data_dir}/Memory/vis"
            Path(dir_path).mkdir(parents=True, exist_ok=True)
        self.load_to_mem_sync()
        for name, kb in self.get_memory().memories.items():
            self.print(f"Saving to {name}.html with {len(kb.concept_extractor.concept_graph.concepts)} concepts")
            await kb.vis(output_file=f"{dir_path}/{name}.html")
        return dir_path

    async def host_agent_ui(
        self,
        agent,
        host: str = "0.0.0.0",
        port: int | None = None,
        access: str = 'local',
        registry_server: str | None = None,
        public_name: str | None = None,
        description: str | None = None,
        use_builtin_server: bool = None
    ) -> dict[str, str]:
        """
        Unified agent hosting with WebSocket-enabled UI and optional registry publishing.

        Args:
            agent: Agent or Chain instance to host
            host: Host address (default: 0.0.0.0 for remote access)
            port: Port number (auto-assigned if None)
            access: 'local', 'remote', or 'registry'
            registry_server: Registry server URL for publishing (e.g., "ws://localhost:8080/ws/registry/connect")
            public_name: Public name for registry publishing
            description: Description for registry publishing
            use_builtin_server: Use toolbox built-in server vs standalone Python server

        Returns:
            Dictionary with access URLs and configuration
        """
        use_builtin_server = use_builtin_server or self.app.is_server
        if not hasattr(self, '_hosted_agents'):
            self._hosted_agents = {}

        agent_id = f"agent_{secrets.token_urlsafe(8)}"

        # Generate unique port if not specified
        if not port:
            port = 8765 + len(self._hosted_agents)

        # Store agent reference
        self._hosted_agents[agent_id] = {
            'agent': agent,
            'port': port,
            'host': host,
            'access': access,
            'public_name': public_name or f"Agent_{agent_id}",
            'description': description
        }

        result = {
            'agent_id': agent_id,
            'local_url': f"http://{host}:{port}",
            'status': 'starting'
        }

        if use_builtin_server:
            # Use toolbox built-in server
            result.update(await self._setup_builtin_server_hosting(agent_id, agent, host, port))
        else:
            # Use standalone Python server
            result.update(await self._setup_standalone_server_hosting(agent_id, agent, host, port))

        # Handle registry publishing if requested
        if access in ['remote', 'registry'] and registry_server:
            if not public_name:
                raise ValueError("public_name required for registry publishing")

            registry_result = await self._publish_to_registry(
                agent=agent,
                public_name=public_name,
                registry_server=registry_server,
                description=description,
                agent_id=agent_id
            )
            result.update(registry_result)

        self.app.print(f"🚀 Agent '{result.get('public_name', agent_id)}' hosted successfully!")
        self.app.print(f"   Local UI: {result['local_url']}")
        if 'public_url' in result:
            self.app.print(f"   Public URL: {result['public_url']}")
            self.app.print(f"   API Key: {result.get('api_key', 'N/A')}")

        return result

    # toolboxv2/mods/isaa/__init__.py - Missing Methods

    import asyncio
    import json
    import secrets
    import threading
    import time
    from concurrent.futures import ThreadPoolExecutor
    from http.server import BaseHTTPRequestHandler, HTTPServer
    from urllib.parse import parse_qs, urlparse


    async def _handle_reset_context(self, agent_id: str, agent, conn_id: str):
        """Handle context reset requests from WebSocket UI."""

        try:
            # Reset agent context if supported
            if hasattr(agent, 'clear_context'):
                agent.clear_context()
                message = "Context reset successfully"
                success = True
            else:
                message = "Agent does not support context reset"
                success = False

            # Send response back to UI
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'reset_response',
                'data': {
                    'success': success,
                    'message': message,
                    'timestamp': time.time()
                }
            })

            self.app.print(f"Context reset requested for agent {agent_id}: {message}")

        except Exception as e:
            error_message = f"Context reset failed: {str(e)}"
            self.app.print(f"Context reset error for agent {agent_id}: {e}")

            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'error',
                'data': {
                    'error': error_message,
                    'timestamp': time.time()
                }
            })

    async def _handle_get_status(self, agent_id: str, agent, conn_id: str):
        """Handle status requests from WebSocket UI."""

        try:
            # Collect agent status information
            status_info = {
                'agent_id': agent_id,
                'agent_name': getattr(agent, 'name', 'Unknown'),
                'agent_type': agent.__class__.__name__,
                'status': 'active',
                'timestamp': time.time(),
                'server_type': 'builtin'
            }

            # Add additional status if available
            if hasattr(agent, 'status'):
                try:
                    agent_status = agent.status()
                    if isinstance(agent_status, dict):
                        status_info.update(agent_status)
                except:
                    pass

            # Add hosted agent info
            if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
                hosted_info = self._hosted_agents[agent_id]
                status_info.update({
                    'host': hosted_info.get('host'),
                    'port': hosted_info.get('port'),
                    'access': hosted_info.get('access'),
                    'public_name': hosted_info.get('public_name')
                })

            # Send status back to UI
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'status_response',
                'data': status_info
            })

            self.app.print(f"Status requested for agent {agent_id}")

        except Exception as e:
            error_message = f"Status retrieval failed: {str(e)}"
            self.app.print(f"Status error for agent {agent_id}: {e}")

            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'error',
                'data': {
                    'error': error_message,
                    'timestamp': time.time()
                }
            })


    async def stop_hosted_agent(self, agent_id: str = None, port: int = None):
        """Stop a hosted agent by agent_id or port."""

        if not hasattr(self, '_hosted_agents') and not hasattr(self, '_standalone_servers'):
            self.app.print("No hosted agents found")
            return False

        # Stop by agent_id
        if agent_id:
            if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
                agent_info = self._hosted_agents[agent_id]
                agent_port = agent_info.get('port')

                # Stop standalone server if exists
                if hasattr(self, '_standalone_servers') and agent_port in self._standalone_servers:
                    server_info = self._standalone_servers[agent_port]
                    try:
                        server_info['server'].shutdown()
                        self.app.print(f"Stopped standalone server for agent {agent_id}")
                    except:
                        pass

                # Clean up hosted agent info
                del self._hosted_agents[agent_id]
                self.app.print(f"Stopped hosted agent {agent_id}")
                return True

        # Stop by port
        if port:
            if hasattr(self, '_standalone_servers') and port in self._standalone_servers:
                server_info = self._standalone_servers[port]
                try:
                    server_info['server'].shutdown()
                    self.app.print(f"Stopped server on port {port}")
                    return True
                except Exception as e:
                    self.app.print(f"Failed to stop server on port {port}: {e}")
                    return False

        self.app.print("Agent or port not found")
        return False

    async def list_hosted_agents(self) -> dict[str, Any]:
        """List all currently hosted agents."""

        hosted_info = {
            'builtin_agents': {},
            'standalone_agents': {},
            'total_count': 0
        }

        # Built-in server agents
        if hasattr(self, '_hosted_agents'):
            for agent_id, info in self._hosted_agents.items():
                hosted_info['builtin_agents'][agent_id] = {
                    'public_name': info.get('public_name'),
                    'host': info.get('host'),
                    'port': info.get('port'),
                    'access': info.get('access'),
                    'description': info.get('description')
                }

        # Standalone server agents
        if hasattr(self, '_standalone_servers'):
            for port, info in self._standalone_servers.items():
                hosted_info['standalone_agents'][info['agent_id']] = {
                    'port': port,
                    'thread_alive': info['thread'].is_alive(),
                    'server_type': 'standalone'
                }

        hosted_info['total_count'] = len(hosted_info['builtin_agents']) + len(hosted_info['standalone_agents'])

        return hosted_info

    def _create_agent_ws_connect_handler(self, agent_id: str):
        """Create WebSocket connect handler for specific agent."""

        async def on_connect(app, conn_id: str, session: dict):
            if not hasattr(self, '_agent_connections'):
                self._agent_connections = {}

            if agent_id not in self._agent_connections:
                self._agent_connections[agent_id] = set()

            self._agent_connections[agent_id].add(conn_id)

            # Send initial status
            await app.ws_send(conn_id, {
                'event': 'agent_connected',
                'data': {
                    'agent_id': agent_id,
                    'status': 'ready',
                    'capabilities': ['chat', 'progress_tracking', 'real_time_updates']
                }
            })

            self.app.print(f"UI client connected to agent {agent_id}: {conn_id}")

        return on_connect

    def _create_agent_ws_message_handler(self, agent_id: str, agent):
        """Create WebSocket message handler for specific agent."""

        async def on_message(app, conn_id: str, session: dict, payload: dict):
            event = payload.get('event')
            data = payload.get('data', {})

            if event == 'chat_message':
                await self._handle_chat_message(agent_id, agent, conn_id, data)
            elif event == 'reset_context':
                await self._handle_reset_context(agent_id, agent, conn_id)
            elif event == 'get_status':
                await self._handle_get_status(agent_id, agent, conn_id)
            else:
                self.app.print(f"Unknown event from UI: {event}")

        return on_message

    def _create_agent_ws_disconnect_handler(self, agent_id: str):
        """Create WebSocket disconnect handler for specific agent."""

        async def on_disconnect(app, conn_id: str, session: dict = None):
            if hasattr(self, '_agent_connections') and agent_id in self._agent_connections:
                self._agent_connections[agent_id].discard(conn_id)

            self.app.print(f"UI client disconnected from agent {agent_id}: {conn_id}")

        return on_disconnect


    async def _broadcast_to_agent_ui(self, agent_id: str, message: dict):
        """Broadcast message to all UI clients connected to specific agent."""
        if not hasattr(self, '_agent_connections') or agent_id not in self._agent_connections:
            return

        for conn_id in self._agent_connections[agent_id].copy():
            try:
                await self.app.ws_send(conn_id, message)
            except Exception as e:
                self.app.print(f"Failed to send to UI client {conn_id}: {e}")
                self._agent_connections[agent_id].discard(conn_id)

    async def _publish_to_registry(
        self,
        agent,
        public_name: str,
        registry_server: str,
        description: str | None = None,
        agent_id: str | None = None
    ) -> dict[str, str]:
        """Publish agent to registry server."""
        try:
            # Import registry client dynamically to avoid circular imports
            registry_client_module = __import__("toolboxv2.mods.registry.client", fromlist=["get_registry_client"])
            get_registry_client = registry_client_module.get_registry_client

            client = get_registry_client(self.app)

            # Connect if not already connected
            if not client.ws or not client.ws.open:
                await client.connect(registry_server)

            if not client.ws or not client.ws.open:
                raise Exception("Failed to connect to registry server")

            # Register the agent
            reg_info = await client.register(agent, public_name, description)

            if reg_info:
                return {
                    'public_url': reg_info.public_url,
                    'api_key': reg_info.public_api_key,
                    'public_agent_id': reg_info.public_agent_id,
                    'registry_status': 'published'
                }
            else:
                raise Exception("Registration failed")

        except Exception as e:
            self.app.print(f"Registry publishing failed: {e}")
            return {'registry_status': 'failed', 'registry_error': str(e)}

    def _get_enhanced_agent_ui_html(self, agent_id: str) -> str:
        """Get production-ready enhanced UI HTML with comprehensive progress visualization."""
        agent_info = self._hosted_agents.get(agent_id, {})
        server_info = {
            'server_type': 'standalone' if not hasattr(self.app, 'tb') else 'builtin',
            'agent_id': agent_id
        }

        # Update the JavaScript section in the HTML template:
        js_config = f"""
                window.SERVER_CONFIG = {json.dumps(server_info)};
            """
        html_template = """<!DOCTYPE html>
    <html lang="en">
    <head>
        <meta charset="UTF-8">
        <meta name="viewport" content="width=device-width, initial-scale=1.0">
        <title>{agent_name}</title>
        <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
        <style>
            :root {
                --bg-primary: #0d1117;
                --bg-secondary: #161b22;
                --bg-tertiary: #21262d;
                --text-primary: #f0f6fc;
                --text-secondary: #8b949e;
                --text-muted: #6e7681;
                --accent-blue: #58a6ff;
                --accent-green: #3fb950;
                --accent-red: #f85149;
                --accent-orange: #d29922;
                --accent-purple: #a5a5f5;
                --accent-cyan: #39d0d8;
                --border-color: #30363d;
                --shadow: 0 2px 8px rgba(0, 0, 0, 0.3);
            }

            * { margin: 0; padding: 0; box-sizing: border-box; }

            body {
                font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
                background: var(--bg-primary);
                color: var(--text-primary);
                height: 100vh;
                display: flex;
                flex-direction: column;
                overflow: hidden;
            }

            .header {
                background: var(--bg-tertiary);
                padding: 12px 20px;
                border-bottom: 1px solid var(--border-color);
                display: flex;
                align-items: center;
                justify-content: space-between;
                box-shadow: var(--shadow);
                z-index: 100;
            }

            .agent-info {
                display: flex;
                align-items: center;
                gap: 16px;
            }

            .agent-title {
                font-size: 18px;
                font-weight: 600;
                color: var(--accent-blue);
            }

            .agent-status {
                display: flex;
                align-items: center;
                gap: 8px;
                font-size: 14px;
            }

            .status-dot {
                width: 10px;
                height: 10px;
                border-radius: 50%;
                background: var(--accent-red);
                animation: pulse 2s infinite;
            }

            .status-dot.connected {
                background: var(--accent-green);
                animation: none;
            }

            .status-dot.processing {
                background: var(--accent-orange);
                animation: pulse 1s infinite;
            }

            @keyframes pulse {
                0%, 100% { opacity: 1; }
                50% { opacity: 0.5; }
            }

            .main-container {
                display: grid;
                grid-template-columns: 2fr 1.5fr 1fr;
                grid-template-rows: 1fr 1fr;
                grid-template-areas:
                    "chat outline activity"
                    "chat system graph";
                flex: 1;
                gap: 1px;
                background: var(--border-color);
                overflow: hidden;
            }

            .panel {
                background: var(--bg-secondary);
                display: flex;
                flex-direction: column;
                overflow: hidden;
            }

            .chat-panel { grid-area: chat; }
            .outline-panel { grid-area: outline; }
            .activity-panel { grid-area: activity; }
            .system-panel { grid-area: system; }
            .graph-panel { grid-area: graph; }

            .panel-header {
                padding: 12px 16px;
                background: var(--bg-tertiary);
                border-bottom: 1px solid var(--border-color);
                font-weight: 600;
                font-size: 12px;
                text-transform: uppercase;
                letter-spacing: 0.5px;
                display: flex;
                align-items: center;
                gap: 8px;
            }

            .panel-content {
                flex: 1;
                overflow-y: auto;
                padding: 12px;
            }

            /* Chat Panel Styles */
            .chat-messages {
                flex: 1;
                overflow-y: auto;
                padding: 16px;
                display: flex;
                flex-direction: column;
                gap: 16px;
            }

            .message {
                display: flex;
                align-items: flex-start;
                gap: 12px;
                max-width: 85%;
            }

            .message.user {
                flex-direction: row-reverse;
                margin-left: auto;
            }

            .message-avatar {
                width: 32px;
                height: 32px;
                border-radius: 50%;
                display: flex;
                align-items: center;
                justify-content: center;
                font-size: 12px;
                font-weight: 600;
                flex-shrink: 0;
            }

            .message.user .message-avatar {
                background: var(--accent-blue);
            }

            .message.agent .message-avatar {
                background: var(--accent-green);
            }

            .message-content {
                padding: 12px 16px;
                border-radius: 12px;
                line-height: 1.5;
                font-size: 14px;
            }

            .message.user .message-content {
                background: var(--accent-blue);
                color: white;
            }

            .message.agent .message-content {
                background: var(--bg-tertiary);
                border: 1px solid var(--border-color);
            }

            .chat-input-area {
                border-top: 1px solid var(--border-color);
                padding: 16px;
                display: flex;
                gap: 12px;
            }

            .chat-input {
                flex: 1;
                background: var(--bg-primary);
                border: 1px solid var(--border-color);
                border-radius: 8px;
                padding: 12px;
                color: var(--text-primary);
                font-size: 14px;
            }

            .chat-input:focus {
                outline: none;
                border-color: var(--accent-blue);
            }

            .send-button {
                background: var(--accent-blue);
                color: white;
                border: none;
                border-radius: 8px;
                padding: 12px 20px;
                cursor: pointer;
                font-weight: 600;
                transition: all 0.2s;
            }

            .send-button:hover:not(:disabled) {
                background: #4493f8;
                transform: translateY(-1px);
            }

            .send-button:disabled {
                opacity: 0.5;
                cursor: not-allowed;
                transform: none;
            }

            /* Progress Indicator */
            .progress-indicator {
                display: none;
                align-items: center;
                gap: 12px;
                padding: 12px 16px;
                background: var(--bg-tertiary);
                border-top: 1px solid var(--border-color);
                font-size: 14px;
            }

            .progress-indicator.active { display: flex; }

            .spinner {
                width: 16px;
                height: 16px;
                border: 2px solid var(--border-color);
                border-top: 2px solid var(--accent-blue);
                border-radius: 50%;
                animation: spin 1s linear infinite;
            }

            @keyframes spin {
                0% { transform: rotate(0deg); }
                100% { transform: rotate(360deg); }
            }

            /* Outline Panel Styles */
            .outline-progress {
                margin-bottom: 16px;
            }

            .outline-header {
                display: flex;
                align-items: center;
                justify-content: space-between;
                margin-bottom: 12px;
            }

            .outline-title {
                font-weight: 600;
                color: var(--accent-cyan);
            }

            .outline-stats {
                font-size: 12px;
                color: var(--text-muted);
            }

            .progress-bar {
                width: 100%;
                height: 6px;
                background: var(--bg-primary);
                border-radius: 3px;
                overflow: hidden;
                margin-bottom: 16px;
            }

            .progress-fill {
                height: 100%;
                background: linear-gradient(90deg, var(--accent-blue), var(--accent-cyan));
                width: 0%;
                transition: width 0.5s ease;
            }

            .outline-steps {
                display: flex;
                flex-direction: column;
                gap: 8px;
            }

            .outline-step {
                display: flex;
                align-items: center;
                gap: 10px;
                padding: 8px 12px;
                border-radius: 6px;
                background: var(--bg-primary);
                border-left: 3px solid var(--border-color);
                transition: all 0.3s;
            }

            .outline-step.active {
                border-left-color: var(--accent-orange);
                background: rgba(217, 153, 34, 0.1);
            }

            .outline-step.completed {
                border-left-color: var(--accent-green);
                background: rgba(63, 185, 80, 0.1);
            }

            .step-icon {
                font-size: 14px;
                width: 16px;
            }

            .step-text {
                flex: 1;
                font-size: 13px;
            }

            .step-method {
                font-size: 11px;
                color: var(--text-muted);
                background: var(--bg-tertiary);
                padding: 2px 6px;
                border-radius: 4px;
            }

            /* Activity Panel Styles */
            .current-activity {
                background: var(--bg-primary);
                border: 1px solid var(--border-color);
                border-radius: 6px;
                padding: 12px;
                margin-bottom: 12px;
            }

            .activity-header {
                display: flex;
                align-items: center;
                gap: 8px;
                margin-bottom: 8px;
            }

            .activity-title {
                font-weight: 600;
                color: var(--accent-orange);
            }

            .activity-duration {
                font-size: 11px;
                color: var(--text-muted);
                background: var(--bg-tertiary);
                padding: 2px 6px;
                border-radius: 4px;
            }

            .activity-description {
                font-size: 13px;
                line-height: 1.4;
                color: var(--text-secondary);
            }

            .meta-tools-list {
                display: flex;
                flex-direction: column;
                gap: 6px;
            }

            .meta-tool {
                display: flex;
                align-items: center;
                gap: 8px;
                padding: 6px 10px;
                background: var(--bg-primary);
                border-radius: 4px;
                font-size: 12px;
            }

            .tool-icon {
                width: 12px;
                text-align: center;
            }

            .tool-name {
                flex: 1;
                color: var(--text-secondary);
            }

            .tool-status {
                font-size: 10px;
                padding: 2px 6px;
                border-radius: 3px;
            }

            .tool-status.running {
                background: var(--accent-orange);
                color: white;
            }

            .tool-status.completed {
                background: var(--accent-green);
                color: white;
            }

            .tool-status.error {
                background: var(--accent-red);
                color: white;
            }

            /* System Panel Styles */
            .system-grid {
                display: grid;
                grid-template-columns: 1fr 2fr;
                gap: 8px 12px;
                font-size: 12px;
            }

            .system-key {
                color: var(--text-muted);
                font-weight: 500;
            }

            .system-value {
                color: var(--text-primary);
                font-family: 'SF Mono', Monaco, monospace;
                word-break: break-word;
            }

            .current-node {
                background: var(--bg-primary);
                padding: 8px 10px;
                border-radius: 6px;
                margin-bottom: 12px;
                border: 1px solid var(--border-color);
            }

            .node-name {
                font-weight: 600;
                color: var(--accent-purple);
                margin-bottom: 4px;
            }

            .node-operation {
                font-size: 11px;
                color: var(--text-muted);
            }

            /* Graph Panel Styles */
            .agent-graph {
                display: flex;
                flex-direction: column;
                align-items: center;
                gap: 8px;
                padding: 8px;
            }

            .graph-node {
                padding: 6px 12px;
                background: var(--bg-primary);
                border: 1px solid var(--border-color);
                border-radius: 6px;
                font-size: 11px;
                text-align: center;
                min-width: 80px;
            }

            .graph-node.active {
                border-color: var(--accent-orange);
                background: rgba(217, 153, 34, 0.1);
            }

            .graph-node.completed {
                border-color: var(--accent-green);
                background: rgba(63, 185, 80, 0.1);
            }

            .graph-arrow {
                color: var(--text-muted);
                font-size: 12px;
            }

            /* Connection Error Styles */
            .connection-error {
                background: var(--accent-red);
                color: white;
                padding: 8px 12px;
                margin: 8px;
                border-radius: 6px;
                font-size: 12px;
                text-align: center;
            }

            .fallback-mode {
                background: var(--accent-orange);
                color: white;
                padding: 8px 12px;
                margin: 8px;
                border-radius: 6px;
                font-size: 12px;
                text-align: center;
            }
        </style>
    </head>
    <body>
        <div class="header">
            <div class="agent-info">
                <div class="agent-title">{agent_name}</div>
                <div class="text-secondary">{agent_description}</div>
            </div>
            <div class="agent-status">
                <div class="status-dot" id="status-dot"></div>
                <span id="status-text">Initializing...</span>
            </div>
        </div>

        <div class="main-container">
            <!-- Chat Panel -->
            <div class="panel chat-panel">
                <div class="panel-header">💬 Conversation</div>
                <div class="chat-messages" id="chat-messages">
                    <div class="message agent">
                        <div class="message-avatar">AI</div>
                        <div class="message-content">Hello! I'm ready to help you. What would you like to know?</div>
                    </div>
                </div>
                <div class="progress-indicator" id="progress-indicator">
                    <div class="spinner"></div>
                    <span id="progress-text">Processing...</span>
                </div>
                <div class="chat-input-area">
                    <input type="text" id="chat-input" class="chat-input" placeholder="Type your message...">
                    <button id="send-button" class="send-button">Send</button>
                </div>
            </div>

            <!-- Outline & Progress Panel -->
            <div class="panel outline-panel">
                <div class="panel-header">📋 Execution Outline</div>
                <div class="panel-content">
                    <div class="outline-progress">
                        <div class="outline-header">
                            <div class="outline-title" id="outline-title">Ready</div>
                            <div class="outline-stats" id="outline-stats">0/0 steps</div>
                        </div>
                        <div class="progress-bar">
                            <div class="progress-fill" id="outline-progress-fill"></div>
                        </div>
                    </div>
                    <div class="outline-steps" id="outline-steps">
                        <div class="outline-step">
                            <div class="step-icon">⏳</div>
                            <div class="step-text">Waiting for query...</div>
                        </div>
                    </div>
                    <div class="current-activity" id="current-activity" style="display: none;">
                        <div class="activity-header">
                            <div class="activity-title" id="activity-title">Current Activity</div>
                            <div class="activity-duration" id="activity-duration">0s</div>
                        </div>
                        <div class="activity-description" id="activity-description"></div>
                    </div>
                </div>
            </div>

            <!-- Activity & Meta-Tools Panel -->
            <div class="panel activity-panel">
                <div class="panel-header">⚙️ Meta-Tool Activity</div>
                <div class="panel-content">
                    <div class="meta-tools-list" id="meta-tools-list">
                        <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                            No activity yet
                        </div>
                    </div>
                </div>
            </div>

            <!-- System Status Panel -->
            <div class="panel system-panel">
                <div class="panel-header">🔧 System Status</div>
                <div class="panel-content">
                    <div class="current-node" id="current-node">
                        <div class="node-name" id="node-name">System</div>
                        <div class="node-operation" id="node-operation">Idle</div>
                    </div>
                    <div class="system-grid" id="system-grid">
                        <div class="system-key">Status</div>
                        <div class="system-value">Ready</div>
                        <div class="system-key">Runtime</div>
                        <div class="system-value">0s</div>
                        <div class="system-key">Events</div>
                        <div class="system-value">0</div>
                        <div class="system-key">Errors</div>
                        <div class="system-value">0</div>
                    </div>
                </div>
            </div>

            <!-- Agent Graph Panel -->
            <div class="panel graph-panel">
                <div class="panel-header">🌐 Agent Flow</div>
                <div class="panel-content">
                    <div class="agent-graph" id="agent-graph">
                        <div class="graph-node">LLMReasonerNode</div>
                        <div class="graph-arrow">↓</div>
                        <div class="graph-node">Ready</div>
                    </div>
                </div>
            </div>
        </div>

        <script unSave="true">
            __SERVER_CONFIG__
            class ProductionAgentUI {
                constructor() {
                    this.ws = null;
                    this.isProcessing = false;
                    this.sessionId = 'ui_session_' + Math.random().toString(36).substr(2, 9);
                    this.startTime = null;
                    this.reconnectAttempts = 0;
                    this.maxReconnectAttempts = 10;
                    this.reconnectDelay = 1000;
                    this.useWebSocket = true;
                    this.fallbackMode = false;

                    // Progress tracking
                    this.currentOutline = null;
                    this.currentActivity = null;
                    this.metaTools = new Map();
                    this.systemStatus = {};
                    this.agentGraph = [];
                    this.progressEvents = [];

                    this.elements = {
                        statusDot: document.getElementById('status-dot'),
                        statusText: document.getElementById('status-text'),
                        chatMessages: document.getElementById('chat-messages'),
                        chatInput: document.getElementById('chat-input'),
                        sendButton: document.getElementById('send-button'),
                        progressIndicator: document.getElementById('progress-indicator'),
                        progressText: document.getElementById('progress-text'),

                        // Outline elements
                        outlineTitle: document.getElementById('outline-title'),
                        outlineStats: document.getElementById('outline-stats'),
                        outlineProgressFill: document.getElementById('outline-progress-fill'),
                        outlineSteps: document.getElementById('outline-steps'),
                        currentActivity: document.getElementById('current-activity'),
                        activityTitle: document.getElementById('activity-title'),
                        activityDuration: document.getElementById('activity-duration'),
                        activityDescription: document.getElementById('activity-description'),

                        // Meta-tools elements
                        metaToolsList: document.getElementById('meta-tools-list'),

                        // System elements
                        currentNode: document.getElementById('current-node'),
                        nodeName: document.getElementById('node-name'),
                        nodeOperation: document.getElementById('node-operation'),
                        systemGrid: document.getElementById('system-grid'),

                        // Graph elements
                        agentGraph: document.getElementById('agent-graph')
                    };
                    this.init();
                }


                init() {

                    this.configureAPIPaths();
                    this.setupEventListeners();
                    this.detectServerMode();
                    this.startStatusUpdates();
                }

                configureAPIPaths() {
                    const serverType = window.SERVER_CONFIG?.server_type || 'standalone';

                    if (serverType === 'builtin') {
                        this.apiPaths = {
                            status: '/api/agent_ui/status',
                            run: '/api/agent_ui/run_agent',
                            reset: '/api/agent_ui/reset_context'
                        };
                        this.useWebSocket = true;
                    } else {
                        this.apiPaths = {
                            status: '/api/status',
                            run: '/api/run',
                            reset: '/api/reset'
                        };
                        this.useWebSocket = false;
                        this.enableFallbackMode();
                    }
                }

                setupEventListeners() {
                    this.elements.sendButton.addEventListener('click', () => this.sendMessage());
                    this.elements.chatInput.addEventListener('keypress', (e) => {
                        if (e.key === 'Enter' && !this.isProcessing) {
                            this.sendMessage();
                        }
                    });

                    // Handle page visibility for reconnection
                    document.addEventListener('visibilitychange', () => {
                        if (!document.hidden && (!this.ws || this.ws.readyState === WebSocket.CLOSED)) {
                            this.connectWebSocket();
                        }
                    });
                }

                detectServerMode() {
                    // Use configured paths instead of hardcoded ones
                    fetch(this.apiPaths.status)
                        .then(response => response.json())
                        .then(data => {
                            this.addLogEntry(`Server detected: ${data.server_type || 'standalone'}`, 'info');
                            if (data.server_type === 'builtin' && this.useWebSocket) {
                                this.connectWebSocket();
                            }
                        })
                        .catch(() => {
                            this.addLogEntry('Server detection failed, using fallback mode', 'error');
                            this.enableFallbackMode();
                        });
                }

                connectWebSocket() {
                    if (!this.useWebSocket) return;

                    try {
                        // Construct WebSocket URL more robustly
                        const protocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
                        const wsUrl = `${protocol}//${window.location.host}/ws/agent_ui/connect`;

                        this.addLogEntry(`Attempting WebSocket connection to: ${wsUrl}`);
                        this.ws = new WebSocket(wsUrl);

                        this.ws.onopen = () => {
                            this.reconnectAttempts = 0;
                            this.fallbackMode = false;
                            this.setStatus('connected', 'Connected');
                            this.addLogEntry('WebSocket connected successfully', 'success');
                            this.removeFallbackIndicators();
                        };

                        this.ws.onmessage = (event) => {
                            try {
                                const message = JSON.parse(event.data);
                                this.handleWebSocketMessage(message);
                            } catch (error) {
                                this.addLogEntry(`WebSocket message parse error: ${error.message}`, 'error');
                            }
                        };

                        this.ws.onclose = (event) => {
                            this.setStatus('disconnected', 'Disconnected');
                            this.addLogEntry(`WebSocket disconnected (code: ${event.code})`, 'error');
                            this.scheduleReconnection();
                        };

                        this.ws.onerror = (error) => {
                            this.setStatus('error', 'Connection Error');
                            this.addLogEntry('WebSocket connection error', 'error');
                            this.scheduleReconnection();
                        };

                    } catch (error) {
                        this.addLogEntry(`WebSocket setup error: ${error.message}`, 'error');
                        this.enableFallbackMode();
                    }
                }

                scheduleReconnection() {
                    if (this.reconnectAttempts >= this.maxReconnectAttempts) {
                        this.addLogEntry('Max reconnection attempts reached, enabling fallback mode', 'error');
                        this.enableFallbackMode();
                        return;
                    }

                    this.reconnectAttempts++;
                    const delay = Math.min(this.reconnectDelay * this.reconnectAttempts, 30000);

                    this.setStatus('error', `Reconnecting in ${delay/1000}s (attempt ${this.reconnectAttempts})`);

                    setTimeout(() => {
                        if (!this.ws || this.ws.readyState === WebSocket.CLOSED) {
                            this.connectWebSocket();
                        }
                    }, delay);
                }

                enableFallbackMode() {
                    this.fallbackMode = true;
                    this.useWebSocket = false;
                    this.setStatus('disconnected', 'Fallback Mode (API Only)');
                    this.showFallbackIndicator();
                    this.addLogEntry('WebSocket unavailable - using API fallback mode', 'info');
                }

                showFallbackIndicator() {
                    const indicator = document.createElement('div');
                    indicator.className = 'fallback-mode';
                    indicator.textContent = 'Using API fallback mode - limited real-time updates';
                    indicator.id = 'fallback-indicator';
                    document.body.appendChild(indicator);
                }

                removeFallbackIndicators() {
                    const indicator = document.getElementById('fallback-indicator');
                    if (indicator) {
                        indicator.remove();
                    }
                }

                handleWebSocketMessage(message) {
                    try {
                        switch (message.event) {
                            case 'agent_connected':
                                this.addLogEntry('Agent ready for interaction', 'success');
                                this.updateSystemStatus({
                                    status: 'Connected',
                                    capabilities: message.data.capabilities
                                });
                                break;

                            case 'processing_start':
                                this.setProcessing(true);
                                this.startTime = Date.now();
                                this.addLogEntry(`Processing: ${message.data.query}`, 'progress');
                                this.resetProgressTracking();
                                break;

                            case 'progress_update':
                                this.handleProgressUpdate(message.data);
                                break;

                            case 'outline_update':
                                this.handleOutlineUpdate(message.data);
                                break;

                            case 'meta_tool_update':
                                this.handleMetaToolUpdate(message.data);
                                break;

                            case 'activity_update':
                                this.handleActivityUpdate(message.data);
                                break;

                            case 'system_update':
                                this.handleSystemUpdate(message.data);
                                break;

                            case 'graph_update':
                                this.handleGraphUpdate(message.data);
                                break;

                            case 'chat_response':
                                this.addMessage('agent', message.data.response);
                                this.setProcessing(false);
                                this.addLogEntry('Response completed', 'success');
                                this.showFinalSummary(message.data);
                                break;

                            case 'error':
                                this.addMessage('agent', `Error: ${message.data.error}`);
                                this.setProcessing(false);
                                this.addLogEntry(`Error: ${message.data.error}`, 'error');
                                break;

                            default:
                                console.log('Unhandled WebSocket message:', message);
                        }
                    } catch (error) {
                        this.addLogEntry(`Message handling error: ${error.message}`, 'error');
                    }
                }

                handleProgressUpdate(data) {
                    this.progressEvents.push(data);

                    const progressText = `${data.event_type}: ${data.status || 'processing'}`;
                    this.elements.progressText.textContent = progressText;

                    // Update based on event type
                    if (data.event_type === 'reasoning_loop') {
                        this.addLogEntry(`🧠 Reasoning loop #${data.loop_number || '?'}`, 'reasoning');
                        this.updateCurrentActivity({
                            title: 'Reasoning',
                            description: data.current_focus || 'Deep thinking in progress',
                            duration: data.time_in_activity || 0
                        });
                    } else if (data.event_type === 'meta_tool_call') {
                        this.addLogEntry(`⚙️ Meta-tool: ${data.meta_tool_name || 'unknown'}`, 'meta-tool');
                    } else {
                        this.addLogEntry(`Progress - ${progressText}`, 'progress');
                    }

                    // Update system status
                    this.updateSystemStatus({
                        current_node: data.node_name,
                        current_operation: data.event_type,
                        runtime: this.getRuntime(),
                        events: this.progressEvents.length
                    });
                }

                handleOutlineUpdate(data) {
                    this.currentOutline = data;

                    if (data.outline_created && data.steps) {
                        this.elements.outlineTitle.textContent = 'Execution Outline';

                        const completedCount = (data.completed_steps || []).length;
                        const totalCount = data.total_steps || data.steps.length;

                        this.elements.outlineStats.textContent = `${completedCount}/${totalCount} steps`;

                        // Update progress bar
                        const progress = totalCount > 0 ? (completedCount / totalCount) * 100 : 0;
                        this.elements.outlineProgressFill.style.width = `${progress}%`;

                        // Update steps
                        this.updateOutlineSteps(data.steps, data.current_step, data.completed_steps || []);

                        this.addLogEntry(`Outline progress: ${completedCount}/${totalCount} steps completed`, 'outline');
                    }
                }

                updateOutlineSteps(steps, currentStep, completedSteps) {
                    this.elements.outlineSteps.innerHTML = '';

                    steps.forEach((step, index) => {
                        const stepEl = document.createElement('div');
                        stepEl.className = 'outline-step';

                        const stepId = step.id || (index + 1);
                        let icon = '⏳';

                        if (completedSteps.includes(stepId)) {
                            stepEl.classList.add('completed');
                            icon = '✅';
                        } else if (stepId === currentStep) {
                            stepEl.classList.add('active');
                            icon = '🔄';
                        }

                        stepEl.innerHTML = `
                            <div class="step-icon">${icon}</div>
                            <div class="step-text">${step.description || `Step ${stepId}`}</div>
                            <div class="step-method">${step.method || 'unknown'}</div>
                        `;

                        this.elements.outlineSteps.appendChild(stepEl);
                    });
                }

                handleMetaToolUpdate(data) {
                    const toolId = `${data.meta_tool_name}_${Date.now()}`;
                    const toolData = {
                        name: data.meta_tool_name,
                        status: data.status || 'running',
                        timestamp: Date.now(),
                        phase: data.execution_phase,
                        data: data
                    };

                    this.metaTools.set(toolId, toolData);
                    this.updateMetaToolsList();

                    // Add to log with appropriate icon
                    const statusIcon = data.status === 'completed' ? '✅' :
                                     data.status === 'error' ? '❌' : '⚙️';
                    this.addLogEntry(`${statusIcon} ${data.meta_tool_name}: ${data.status || 'running'}`, 'meta-tool');
                }

                updateMetaToolsList() {
                    this.elements.metaToolsList.innerHTML = '';

                    if (this.metaTools.size === 0) {
                        this.elements.metaToolsList.innerHTML = `
                            <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                                No meta-tool activity yet
                            </div>
                        `;
                        return;
                    }

                    // Show recent meta-tools (last 8)
                    const recentTools = Array.from(this.metaTools.values())
                        .sort((a, b) => b.timestamp - a.timestamp)
                        .slice(0, 8);

                    recentTools.forEach(tool => {
                        const toolEl = document.createElement('div');
                        toolEl.className = 'meta-tool';

                        const icons = {
                            internal_reasoning: '🧠',
                            delegate_to_llm_tool_node: '🎯',
                            create_and_execute_plan: '📋',
                            manage_internal_task_stack: '📚',
                            advance_outline_step: '➡️',
                            write_to_variables: '💾',
                            read_from_variables: '📖',
                            direct_response: '✨'
                        };

                        const icon = icons[tool.name] || '⚙️';
                        const displayName = tool.name.replace(/_/g, ' ');
                        const age = Math.floor((Date.now() - tool.timestamp) / 1000);

                        toolEl.innerHTML = `
                            <div class="tool-icon">${icon}</div>
                            <div class="tool-name">${displayName} (${age}s ago)</div>
                            <div class="tool-status ${tool.status}">${tool.status}</div>
                        `;

                        this.elements.metaToolsList.appendChild(toolEl);
                    });
                }

                handleActivityUpdate(data) {
                    this.currentActivity = data;
                    this.updateCurrentActivity(data);
                }

                updateCurrentActivity(data) {
                    if (data.primary_activity && data.primary_activity !== 'Unknown') {
                        this.elements.currentActivity.style.display = 'block';
                        this.elements.activityTitle.textContent = data.primary_activity || data.title;

                        const duration = data.time_in_current_activity || data.duration || 0;
                        if (duration > 0) {
                            this.elements.activityDuration.textContent = this.formatDuration(duration);
                        }

                        this.elements.activityDescription.textContent =
                            data.detailed_description || data.description || '';
                    } else {
                        this.elements.currentActivity.style.display = 'none';
                    }
                }

                handleSystemUpdate(data) {
                    this.systemStatus = { ...this.systemStatus, ...data };
                    this.updateSystemStatus(data);
                }

                updateSystemStatus(data) {
                    // Update current node
                    if (data.current_node) {
                        this.elements.nodeName.textContent = data.current_node;
                        this.elements.nodeOperation.textContent = data.current_operation || 'Processing';
                    }

                    // Update system grid
                    const gridData = [
                        ['Status', data.status || this.systemStatus.status || 'Running'],
                        ['Runtime', this.formatDuration(data.runtime || this.getRuntime())],
                        ['Events', data.events || this.progressEvents.length],
                        ['Errors', data.error_count || this.systemStatus.error_count || 0],
                        ['Node', data.current_node || this.systemStatus.current_node || 'Unknown']
                    ];

                    if (data.total_cost !== undefined) {
                        gridData.push(['Cost', `$${data.total_cost.toFixed(4)}`]);
                    }

                    if (data.total_tokens !== undefined) {
                        gridData.push(['Tokens', data.total_tokens.toLocaleString()]);
                    }

                    this.elements.systemGrid.innerHTML = '';
                    gridData.forEach(([key, value]) => {
                        this.elements.systemGrid.innerHTML += `
                            <div class="system-key">${key}</div>
                            <div class="system-value">${value}</div>
                        `;
                    });
                }

                handleGraphUpdate(data) {
                    this.agentGraph = data.nodes || [];
                    this.updateAgentGraph();
                }

                updateAgentGraph() {
                    this.elements.agentGraph.innerHTML = '';

                    if (this.agentGraph.length === 0) {
                        const currentNode = this.systemStatus.current_node || 'LLMReasonerNode';
                        this.elements.agentGraph.innerHTML = `
                            <div class="graph-node active">${currentNode}</div>
                            <div class="graph-arrow">↓</div>
                            <div class="graph-node">Processing</div>
                        `;
                        return;
                    }

                    this.agentGraph.forEach((node, index) => {
                        const nodeEl = document.createElement('div');
                        nodeEl.className = 'graph-node';

                        if (node.active) nodeEl.classList.add('active');
                        if (node.completed) nodeEl.classList.add('completed');

                        nodeEl.textContent = node.name || `Node ${index + 1}`;
                        this.elements.agentGraph.appendChild(nodeEl);

                        if (index < this.agentGraph.length - 1) {
                            const arrow = document.createElement('div');
                            arrow.className = 'graph-arrow';
                            arrow.textContent = '↓';
                            this.elements.agentGraph.appendChild(arrow);
                        }
                    });
                }

                async sendMessage() {
                    const message = this.elements.chatInput.value.trim();
                    if (!message || this.isProcessing) return;

                    this.addMessage('user', message);
                    this.elements.chatInput.value = '';

                    if (this.useWebSocket && this.ws && this.ws.readyState === WebSocket.OPEN) {
                        // Send via WebSocket
                        this.ws.send(JSON.stringify({
                            event: 'chat_message',
                            data: {
                                message: message,
                                session_id: this.sessionId
                            }
                        }));
                    } else {
                        // Fallback to API
                        await this.sendMessageViaAPI(message);
                    }
                }

                async sendMessageViaAPI(message) {
                    this.setProcessing(true);
                    this.startTime = Date.now();
                    this.resetProgressTracking();

                    try {
                        const response = await fetch(this.apiPaths.run, {
                            method: 'POST',
                            headers: {
                                'Content-Type': 'application/json'
                            },
                            body: JSON.stringify({
                                query: message,
                                session_id: this.sessionId,
                                include_progress: true
                            })
                        });

                        const result = await response.json();

                        if (result.success) {
                            this.addMessage('agent', result.result);
                            this.addLogEntry(`Request completed via API`, 'success');

                            // Process progress events if available
                            if (result.progress_events) {
                                this.processAPIProgressEvents(result.progress_events);
                            }

                            // Process enhanced progress if available
                            if (result.enhanced_progress) {
                                this.processEnhancedProgress(result.enhanced_progress);
                            }
                        } else {
                            this.addMessage('agent', `Error: ${result.error}`);
                            this.addLogEntry(`API request failed: ${result.error}`, 'error');
                        }

                    } catch (error) {
                        this.addMessage('agent', `Network error: ${error.message}`);
                        this.addLogEntry(`Network error: ${error.message}`, 'error');
                    } finally {
                        this.setProcessing(false);
                    }
                }

                processAPIProgressEvents(events) {
                    events.forEach(event => {
                        this.handleProgressUpdate(event);
                    });
                }

                processEnhancedProgress(progress) {
                    if (progress.outline) {
                        this.handleOutlineUpdate(progress.outline);
                    }
                    if (progress.activity) {
                        this.handleActivityUpdate(progress.activity);
                    }
                    if (progress.system) {
                        this.handleSystemUpdate(progress.system);
                    }
                    if (progress.graph) {
                        this.handleGraphUpdate(progress.graph);
                    }
                }

                resetProgressTracking() {
                    this.progressEvents = [];
                    this.metaTools.clear();
                    this.updateSystemStatus({ status: 'Processing', events: 0 });
                }

                showFinalSummary(data) {
                    if (data.final_summary) {
                        const summary = data.final_summary;
                        this.addLogEntry(`Final Summary - Outline: ${summary.outline_completed ? 'Complete' : 'Partial'}, Meta-tools: ${summary.total_meta_tools}, Nodes: ${summary.total_nodes}`, 'success');
                    }
                }

                addMessage(sender, content) {
                    const messageEl = document.createElement('div');
                    messageEl.classList.add('message', sender);

                    const avatarEl = document.createElement('div');
                    avatarEl.classList.add('message-avatar');
                    avatarEl.textContent = sender === 'user' ? 'You' : 'AI';

                    const contentEl = document.createElement('div');
                    contentEl.classList.add('message-content');

                    if (sender === 'agent' && window.marked) {
                        try {
                            contentEl.innerHTML = marked.parse(content);
                        } catch (error) {
                            contentEl.textContent = content;
                        }
                    } else {
                        contentEl.textContent = content;
                    }

                    messageEl.appendChild(avatarEl);
                    messageEl.appendChild(contentEl);

                    this.elements.chatMessages.appendChild(messageEl);
                    this.elements.chatMessages.scrollTop = this.elements.chatMessages.scrollHeight;
                }

                addLogEntry(message, type = 'info') {
                    // For debugging - could show in a log panel
                    const timestamp = new Date().toLocaleTimeString();
                    console.log(`[${timestamp}] [${type.toUpperCase()}] ${message}`);
                }

                setStatus(status, text) {
                    this.elements.statusDot.className = `status-dot ${status}`;
                    this.elements.statusText.textContent = text;
                }

                setProcessing(processing) {
                    this.isProcessing = processing;
                    this.elements.sendButton.disabled = processing;
                    this.elements.chatInput.disabled = processing;

                    if (processing) {
                        this.elements.progressIndicator.classList.add('active');
                        this.setStatus('processing', 'Processing');
                    } else {
                        this.elements.progressIndicator.classList.remove('active');
                        this.setStatus(this.ws && this.ws.readyState === WebSocket.OPEN ? 'connected' : 'disconnected',
                                      this.ws && this.ws.readyState === WebSocket.OPEN ? 'Connected' : 'Disconnected');
                        this.startTime = null;
                    }
                }

                formatDuration(seconds) {
                    if (typeof seconds !== 'number') return '0s';
                    if (seconds < 60) return `${seconds.toFixed(1)}s`;
                    if (seconds < 3600) return `${Math.floor(seconds/60)}m${Math.floor(seconds%60)}s`;
                    return `${Math.floor(seconds/3600)}h${Math.floor((seconds%3600)/60)}m`;
                }

                getRuntime() {
                    return this.startTime ? (Date.now() - this.startTime) / 1000 : 0;
                }

                startStatusUpdates() {
                    setInterval(() => {
                        if (this.isProcessing) {
                            this.updateSystemStatus({ runtime: this.getRuntime() });
                        }
                    }, 1000);
                }
            }

            // Initialize the production UI
            if (!window.TB) {

                document.addEventListener('DOMContentLoaded', () => {
                    window.agentUI = new ProductionAgentUI();
                });
            } else {
                TB.once(() => {
                    window.agentUI = new ProductionAgentUI();
                });
            }
        </script>
    </body>
    </html>"""

        return (html_template.
                replace("{agent_name}", agent_info.get('public_name', 'Agent Interface')).
                replace("{agent_description}", agent_info.get('description', '')).
                replace("__SERVER_CONFIG__", js_config)
                )

    async def _handle_chat_message_with_progress_integration(self, agent_id: str, agent, conn_id: str, data: dict):
        """Enhanced chat message handler with ProgressiveTreePrinter integration."""
        query = data.get('message', '')
        session_id = data.get('session_id', f"ui_session_{conn_id}")

        if not query:
            return

        # Create ProgressiveTreePrinter for real-time UI updates
        from toolboxv2.mods.isaa.extras.terminal_progress import (
            ProgressiveTreePrinter,
            VerbosityMode,
        )
        progress_printer = ProgressiveTreePrinter(
            mode=VerbosityMode.STANDARD,
            use_rich=False,
            auto_refresh=False
        )

        # Enhanced progress callback that extracts all UI data
        async def comprehensive_progress_callback(event):
            try:
                # Add event to progress printer for processing
                progress_printer.tree_builder.add_event(event)

                # Get comprehensive summary from the printer
                summary = progress_printer.tree_builder.get_execution_summary()

                # Extract outline information
                outline_info = progress_printer._get_current_outline_info()

                # Extract current activity
                activity_info = progress_printer._get_detailed_current_activity()

                # Extract tool usage
                tool_usage = progress_printer._get_tool_usage_summary()

                # Extract task progress
                task_progress = progress_printer._get_task_executor_progress()

                # Send basic progress update
                await self._broadcast_to_agent_ui(agent_id, {
                    'event': 'progress_update',
                    'data': {
                        'event_type': event.event_type,
                        'status': getattr(event, 'status', 'processing').value if hasattr(event, 'status') and event.status else 'unknown',
                        'node_name': getattr(event, 'node_name', 'Unknown'),
                        'timestamp': event.timestamp,
                        'loop_number': getattr(event.metadata, {}).get('reasoning_loop', 0),
                        'meta_tool_name': getattr(event.metadata, {}).get('meta_tool_name'),
                        'current_focus': getattr(event.metadata, {}).get('current_focus', ''),
                        'time_in_activity': activity_info.get('time_in_current_activity', 0)
                    }
                })

                # Send outline updates
                if outline_info.get('outline_created'):
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'outline_update',
                        'data': outline_info
                    })

                # Send meta-tool updates
                if event.metadata and event.metadata.get('meta_tool_name'):
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'meta_tool_update',
                        'data': {
                            'meta_tool_name': event.metadata['meta_tool_name'],
                            'status': 'completed' if event.success else (
                                'error' if event.success is False else 'running'),
                            'execution_phase': event.metadata.get('execution_phase', 'unknown'),
                            'reasoning_loop': event.metadata.get('reasoning_loop', 0),
                            'timestamp': event.timestamp
                        }
                    })

                # Send activity updates
                if activity_info['primary_activity'] != 'Unknown':
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'activity_update',
                        'data': activity_info
                    })

                # Send system updates
                await self._broadcast_to_agent_ui(agent_id, {
                    'event': 'system_update',
                    'data': {
                        'current_node': summary['execution_flow']['current_node'],
                        'current_operation': activity_info.get('primary_activity', 'Processing'),
                        'status': 'Processing',
                        'runtime': summary['timing']['elapsed'],
                        'total_events': summary['performance_metrics']['total_events'],
                        'error_count': summary['performance_metrics']['error_count'],
                        'total_cost': summary['performance_metrics']['total_cost'],
                        'total_tokens': summary['performance_metrics']['total_tokens'],
                        'completed_nodes': summary['session_info']['completed_nodes'],
                        'total_nodes': summary['session_info']['total_nodes'],
                        'tool_usage': {
                            'tools_used': list(tool_usage.get('tools_used', set())),
                            'tools_active': list(tool_usage.get('tools_active', set())),
                            'current_tool_operation': tool_usage.get('current_tool_operation')
                        }
                    }
                })

                # Send graph updates
                flow_nodes = []
                for node_name in summary['execution_flow']['flow']:
                    if node_name in progress_printer.tree_builder.nodes:
                        node = progress_printer.tree_builder.nodes[node_name]
                        flow_nodes.append({
                            'name': node_name,
                            'active': node_name in summary['execution_flow']['active_nodes'],
                            'completed': (node.status.value == 'completed') if node.status else False,
                            'status': node.status.value if node.status else 'unknown'
                        })

                if flow_nodes:
                    await self._broadcast_to_agent_ui(agent_id, {
                        'event': 'graph_update',
                        'data': {'nodes': flow_nodes}
                    })

            except Exception as e:
                self.app.print(f"Comprehensive progress callback error: {e}")

        # Set progress callback
        original_callback = getattr(agent, 'progress_callback', None)

        try:
            if hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(comprehensive_progress_callback)
            elif hasattr(agent, 'progress_callback'):
                agent.progress_callback = comprehensive_progress_callback

            # Send processing start notification
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'processing_start',
                'data': {'query': query, 'session_id': session_id}
            })

            # Execute agent
            result = await agent.a_run(query=query, session_id=session_id)

            # Get final summary
            final_summary = progress_printer.tree_builder.get_execution_summary()

            # Extract outline information
            outline_info = progress_printer._get_current_outline_info()

            # Initialize outline_info if empty
            if not outline_info or not outline_info.get('steps'):
                outline_info = {
                    'steps': [],
                    'current_step': 1,
                    'completed_steps': [],
                    'total_steps': 0,
                    'step_descriptions': {},
                    'current_step_progress': "",
                    'outline_raw_data': None,
                    'outline_created': False,
                    'actual_step_completions': []
                }

            # Try to infer outline from execution pattern if not found
            if not outline_info.get('outline_created'):
                outline_info = progress_printer._infer_outline_from_execution_pattern(outline_info)

            # Send final result with summary
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'chat_response',
                'data': {
                    'response': result,
                    'query': query,
                    'session_id': session_id,
                    'completed_at': asyncio.get_event_loop().time(),
                    'final_summary': {
                        'outline_completed': len(outline_info.get('completed_steps', [])) == outline_info.get(
                            'total_steps', 0),
                        'total_meta_tools': len([e for e in progress_printer.tree_builder.nodes.values()
                                                 for event in e.llm_calls + e.sub_events
                                                 if event.metadata and event.metadata.get('meta_tool_name')]),
                        'total_nodes': final_summary['session_info']['total_nodes'],
                        'execution_time': final_summary['timing']['elapsed'],
                        'total_cost': final_summary['performance_metrics']['total_cost']
                    }
                }
            })

        except Exception as e:
            await self._broadcast_to_agent_ui(agent_id, {
                'event': 'error',
                'data': {'error': str(e), 'query': query}
            })
        finally:
            # Restore original callback
            if hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(original_callback)
            elif hasattr(agent, 'progress_callback'):
                agent.progress_callback = original_callback

    # Replace the existing method
    async def _handle_chat_message(self, agent_id: str, agent, conn_id: str, data: dict):
        """Delegate to enhanced handler."""
        await self._handle_chat_message_with_progress_integration(agent_id, agent, conn_id, data)

    # Unified publish and host method
    # toolboxv2/mods/isaa/Tools.py

    async def publish_and_host_agent(
        self,
        agent,
        public_name: str,
        registry_server: str = "ws://localhost:8080/ws/registry/connect",
        description: str | None = None,
        access_level: str = "public"
    ) -> dict[str, Any]:
        """FIXED: Mit Debug-Ausgaben für Troubleshooting."""

        if hasattr(agent, 'name') and not hasattr(agent, 'amd') and hasattr(agent, 'a_run'):
            agent.amd = lambda :None
            agent.amd.name = agent.name

        try:
            # Registry Client initialisieren
            from toolboxv2.mods.registry.client import get_registry_client
            registry_client = get_registry_client(self.app)

            self.app.print(f"Connecting to registry server: {registry_server}")
            await registry_client.connect(registry_server)

            # Progress Callback für Live-Updates einrichten
            callback_success = await self.setup_live_progress_callback(agent, registry_client, f"agent_{agent.amd.name}")
            if not callback_success:
                self.app.print("Warning: Progress callback setup failed")
            else:
                self.app.print("✅ Progress callback setup successful")

            # Agent beim Registry registrieren
            self.app.print(f"Registering agent: {public_name}")
            registration_info = await registry_client.register(
                agent_instance=agent,
                public_name=public_name,
                description=description or f"Agent: {public_name}"
            )

            if not registration_info:
                return {"error": "Registration failed", "success": False}

            self.app.print(f"✅ Agent registration successful: {registration_info.public_agent_id}")

            result = {
                "success": True,
                "agent_name": public_name,
                "public_agent_id": registration_info.public_agent_id,
                "public_api_key": registration_info.public_api_key,
                "public_url": registration_info.public_url,
                "registry_server": registry_server,
                "access_level": access_level,
                "ui_url": registration_info.public_url.replace("/api/registry/run", "/api/registry/ui"),
                "websocket_url": registry_server.replace("/connect", "/ui_connect"),
                "status": "registered"
            }

            return result

        except Exception as e:
            self.app.print(f"Failed to publish agent: {e}")
            return {"error": str(e), "success": False}

    # toolboxv2/mods/isaa/Tools.py

    async def setup_live_progress_callback(self, agent, registry_client, agent_id: str = None):
        """Enhanced setup for live progress callback with proper error handling."""

        if not registry_client:
            self.app.print("Warning: No registry client provided for progress updates")
            return False

        if not registry_client.is_connected:
            self.app.print("Warning: Registry client is not connected")
            return False

        progress_tracker = EnhancedProgressTracker()

        # Generate agent ID if not provided
        if not agent_id:
            agent_id = getattr(agent, 'name', f'agent_{id(agent)}')

        async def enhanced_live_progress_callback(event: ProgressEvent):
            """Enhanced progress callback with comprehensive data extraction."""
            try:
                # Validate event
                if not event:
                    self.app.print("Warning: Received null progress event")
                    return

                # Debug output for local development
                event_type = getattr(event, 'event_type', 'unknown')
                status = getattr(event, 'status', 'unknown')
                agent_name = getattr(event, 'agent_name', 'Unknown Agent')

                self.app.print(f"📊 Progress Event: {event_type} | {status} | {agent_name}")

                # Extract comprehensive progress data
                progress_data = progress_tracker.extract_progress_data(event)

                # Prepare enhanced progress message
                ui_progress_data = {
                    "agent_id": agent_id,
                    "event_type": event_type,
                    "status": status.value if hasattr(status, 'value') else str(status),
                    "timestamp": getattr(event, 'timestamp', asyncio.get_event_loop().time()),
                    "agent_name": agent_name,
                    "node_name": getattr(event, 'node_name', 'Unknown'),
                    "session_id": getattr(event, 'session_id', None),

                    # Core event metadata
                    "metadata": {
                        **getattr(event, 'metadata', {}),
                        "event_id": getattr(event, 'event_id', f"evt_{asyncio.get_event_loop().time()}"),
                        "sequence_number": getattr(event, 'sequence_number', 0),
                        "parent_event_id": getattr(event, 'parent_event_id', None)
                    },

                    # Detailed progress data for UI panels
                    "progress_data": progress_data,

                    # UI-specific flags for selective updates
                    "ui_flags": {
                        "should_update_outline": bool(progress_data.get('outline')),
                        "should_update_activity": bool(progress_data.get('activity')),
                        "should_update_meta_tools": bool(progress_data.get('meta_tool')),
                        "should_update_system": bool(progress_data.get('system')),
                        "should_update_graph": bool(progress_data.get('graph')),
                        "is_error": event_type.lower() in ['error', 'exception', 'failed'],
                        "is_completion": event_type.lower() in ['complete', 'finished', 'success'],
                        "requires_user_input": getattr(event, 'requires_user_input', False)
                    },

                    # Performance metrics
                    "performance": {
                        "execution_time": getattr(event, 'execution_time', None),
                        "memory_delta": getattr(event, 'memory_delta', None),
                        "tokens_used": getattr(event, 'tokens_used', None),
                        "api_calls_made": getattr(event, 'api_calls_made', None)
                    }
                }

                # Send live update to registry server
                await registry_client.send_ui_progress(ui_progress_data)

                # Also send agent status update if this is a significant event
                if event_type in ['started', 'completed', 'error', 'paused', 'resumed']:
                    agent_status = 'processing'
                    if event_type == 'completed':
                        agent_status = 'idle'
                    elif event_type == 'error':
                        agent_status = 'error'
                    elif event_type == 'paused':
                        agent_status = 'paused'

                    await registry_client.send_agent_status(
                        agent_id=agent_id,
                        status=agent_status,
                        details={
                            "last_event": event_type,
                            "last_update": ui_progress_data["timestamp"],
                            "current_node": progress_data.get('graph', {}).get('current_node', 'Unknown')
                        }
                    )

                # Log successful progress update
                self.app.print(f"✅ Sent progress update: {event_type} -> Registry Server")

            except Exception as e:
                self.app.print(f"❌ Progress callback error: {e}")
                # Send error notification to UI
                try:
                    await registry_client.send_ui_progress({
                        "agent_id": agent_id,
                        "event_type": "progress_callback_error",
                        "status": "error",
                        "timestamp": asyncio.get_event_loop().time(),
                        "agent_name": getattr(agent, 'name', 'Unknown'),
                        "metadata": {"error": str(e)},
                        "ui_flags": {"is_error": True}
                    })
                except Exception as nested_error:
                    self.app.print(f"Failed to send error notification: {nested_error}")

        # Set up progress callback with enhanced error handling
        callback_set = False

        if hasattr(agent, 'set_progress_callback'):
            try:
                self.app.print(f"🔧 Setting progress callback via set_progress_callback for agent: {agent_id}")
                agent.set_progress_callback(enhanced_live_progress_callback)
                callback_set = True
            except Exception as e:
                self.app.print(f"Failed to set progress callback via set_progress_callback: {e}")

        if not callback_set and hasattr(agent, 'progress_callback'):
            try:
                self.app.print(f"🔧 Setting progress callback via direct assignment for agent: {agent_id}")
                agent.progress_callback = enhanced_live_progress_callback
                callback_set = True
            except Exception as e:
                self.app.print(f"Failed to set progress callback via direct assignment: {e}")

        if not callback_set:
            self.app.print(f"⚠️ Warning: Agent {agent_id} doesn't support progress callbacks")
            return False

        # Send initial agent status
        try:
            await registry_client.send_agent_status(
                agent_id=agent_id,
                status='online',
                details={
                    "progress_callback_enabled": True,
                    "callback_setup_time": asyncio.get_event_loop().time(),
                    "agent_type": type(agent).__name__
                }
            )
            self.app.print(f"✅ Progress callback successfully set up for agent: {agent_id}")
        except Exception as e:
            self.app.print(f"Failed to send initial agent status: {e}")

        return True


    async def _setup_builtin_server_hosting(self, agent_id: str, agent, host, port) -> dict[str, str]:
        """Setup agent hosting using toolbox built-in server with enhanced WebSocket support."""

        # Register WebSocket handlers for this agent
        @self.app.tb(mod_name="agent_ui", websocket_handler="connect")
        def register_agent_ws_handlers(_):
            return {
                "on_connect": self._create_agent_ws_connect_handler(agent_id),
                "on_message": self._create_agent_ws_message_handler(agent_id, agent),
                "on_disconnect": self._create_agent_ws_disconnect_handler(agent_id),
            }

        # Register UI endpoint - now uses enhanced UI
        @self.app.tb(mod_name="agent_ui", api=True, version="1", api_methods=['GET'])
        async def ui():
            return Result.html(
                self._get_enhanced_agent_ui_html(agent_id), row=True
            )

        # Register API endpoint for direct agent interaction
        @self.app.tb(mod_name="agent_ui", api=True, version="1", request_as_kwarg=True, api_methods=['POST'])
        async def run_agent(request: RequestData):
            return await self._handle_direct_agent_run(agent_id, agent, request)

        # Register additional API endpoints for enhanced features
        @self.app.tb(mod_name="agent_ui", api=True, version="1", request_as_kwarg=True, api_methods=['POST'])
        async def reset_context(request: RequestData):
            return await self._handle_api_reset_context(agent_id, agent, request)

        @self.app.tb(mod_name="agent_ui", api=True, version="1", request_as_kwarg=True, api_methods=['GET'])
        async def status(request: RequestData):
            return await self._handle_api_get_status(agent_id, agent, request)

        # WebSocket endpoint URL
        uri = f"{host}:{port}" if port else f"{host}"
        ws_url = f"ws://{uri}/ws/agent_ui/connect"
        ui_url = f"http://{uri}/api/agent_ui/ui"
        api_url = f"http://{uri}/api/agent_ui/run_agent"

        return {
            'ui_url': ui_url,
            'ws_url': ws_url,
            'api_url': api_url,
            'reset_url': f"http://localhost:{self.app.args_sto.port}/api/agent_ui/reset_context",
            'status_url': f"http://localhost:{self.app.args_sto.port}/api/agent_ui/status",
            'server_type': 'builtin',
            'status': 'running'
        }

    async def _setup_standalone_server_hosting(self, agent_id: str, agent, host: str, port: int) -> dict[str, str]:
        """Setup agent hosting using standalone Python HTTP server with enhanced UI support."""

        if not hasattr(self, '_standalone_servers'):
            self._standalone_servers = {}

        if port in self._standalone_servers:
            self.app.print(f"Port {port} is already in use by another agent")
            return {'status': 'error', 'error': f'Port {port} already in use'}

        # Store server info for the handler
        server_info = {
            'agent_id': agent_id,
            'server_type': 'standalone',
            'api_paths': {
                'ui': '/ui',
                'status': '/api/status',
                'run': '/api/run',
                'reset': '/api/reset'
            }
        }

        # Create handler factory with agent reference and server info
        def handler_factory(*args, **kwargs):
            handler = EnhancedAgentRequestHandler(self, agent_id, agent, *args, **kwargs)
            handler.server_info = server_info
            return handler

        # Start HTTP server in separate thread
        def run_server():
            try:
                httpd = HTTPServer((host, port), handler_factory)
                self._standalone_servers[port] = {
                    'server': httpd,
                    'agent_id': agent_id,
                    'thread': threading.current_thread(),
                    'server_info': server_info
                }

                self.app.print(f"Enhanced standalone server for agent '{agent_id}' running on http://{host}:{port}")
                self.app.print(f"  UI: http://{host}:{port}/ui")
                self.app.print(f"  API: http://{host}:{port}/api/run")
                self.app.print(f"  Status: http://{host}:{port}/api/status")

                httpd.serve_forever()

            except Exception as e:
                self.app.print(f"Standalone server failed: {e}")
            finally:
                if port in self._standalone_servers:
                    del self._standalone_servers[port]

        # Start server in daemon thread
        server_thread = threading.Thread(target=run_server, daemon=True)
        server_thread.start()

        # Wait a moment to ensure server starts
        await asyncio.sleep(0.5)

        return {
            'server_type': 'standalone',
            'local_url': f"http://{host}:{port}",
            'ui_url': f"http://{host}:{port}/ui",
            'api_url': f"http://{host}:{port}/api/run",
            'reset_url': f"http://{host}:{port}/api/reset",
            'status_url': f"http://{host}:{port}/api/status",
            'status': 'running',
            'port': port
        }

    async def _handle_direct_agent_run(self, agent_id: str, agent, request_data) -> Result:
        """Handle direct agent API calls with enhanced progress tracking."""

        try:
            # Parse request body
            body = request_data.body if hasattr(request_data, 'body') else {}

            if not isinstance(body, dict):
                return Result.default_user_error("Request body must be JSON object", exec_code=400)

            query = body.get('query', '')
            session_id = body.get('session_id', f'api_{secrets.token_hex(8)}')
            kwargs = body.get('kwargs', {})
            include_progress = body.get('include_progress', True)

            if not query:
                return Result.default_user_error("Missing 'query' field in request body", exec_code=400)

            # Enhanced progress tracking for API
            progress_events = []
            enhanced_progress = {}

            async def enhanced_api_progress_callback(event):
                if include_progress:
                    progress_tracker = EnhancedProgressTracker()
                    progress_data = progress_tracker.extract_progress_data(event)

                    progress_events.append({
                        'timestamp': event.timestamp,
                        'event_type': event.event_type,
                        'status': event.status.value if event.status else 'unknown',
                        'agent_name': event.agent_name,
                        'metadata': event.metadata
                    })

                    # Store enhanced progress data
                    enhanced_progress.update(progress_data)

            # Set progress callback
            original_callback = getattr(agent, 'progress_callback', None)

            try:
                if hasattr(agent, 'set_progress_callback'):
                    agent.set_progress_callback(enhanced_api_progress_callback)
                elif hasattr(agent, 'progress_callback'):
                    agent.progress_callback = enhanced_api_progress_callback

                # Execute agent
                result = await agent.a_run(query=query, session_id=session_id, **kwargs)

                # Return enhanced structured response
                response_data = {
                    'success': True,
                    'result': result,
                    'session_id': session_id,
                    'agent_id': agent_id,
                    'execution_time': time.time()
                }

                if include_progress:
                    response_data.update({
                        'progress_events': progress_events,
                        'enhanced_progress': enhanced_progress,
                        'outline_info': enhanced_progress.get('outline', {}),
                        'system_info': enhanced_progress.get('system', {}),
                        'meta_tools_used': enhanced_progress.get('meta_tools', [])
                    })

                return Result.json(data=response_data)

            except Exception as e:
                self.app.print(f"Agent execution error: {e}")
                return Result.default_internal_error(
                    info=f"Agent execution failed: {str(e)}",
                    exec_code=500
                )
            finally:
                # Restore original callback
                if hasattr(agent, 'set_progress_callback'):
                    agent.set_progress_callback(original_callback)
                elif hasattr(agent, 'progress_callback'):
                    agent.progress_callback = original_callback

        except Exception as e:
            self.app.print(f"Direct agent run error: {e}")
            return Result.default_internal_error(
                info=f"Request processing failed: {str(e)}",
                exec_code=500
            )

    async def _handle_api_reset_context(self, agent_id: str, agent, request_data) -> Result:
        """Handle API context reset requests."""
        try:
            if hasattr(agent, 'clear_context'):
                agent.clear_context()
                message = "Context reset successfully"
                success = True
            elif hasattr(agent, 'reset'):
                agent.reset()
                message = "Agent reset successfully"
                success = True
            else:
                message = "Agent does not support context reset"
                success = False

            return Result.json(data={
                'success': success,
                'message': message,
                'agent_id': agent_id,
                'timestamp': time.time()
            })

        except Exception as e:
            return Result.default_internal_error(
                info=f"Context reset failed: {str(e)}",
                exec_code=500
            )

    async def _handle_api_get_status(self, agent_id: str, agent, request_data) -> Result:
        """Handle API status requests."""
        try:
            # Collect comprehensive agent status
            status_info = {
                'agent_id': agent_id,
                'agent_name': getattr(agent, 'name', 'Unknown'),
                'agent_type': agent.__class__.__name__,
                'status': 'active',
                'timestamp': time.time(),
                'server_type': 'api'
            }

            # Add agent-specific status
            if hasattr(agent, 'status'):
                try:
                    agent_status = agent.status()
                    if isinstance(agent_status, dict):
                        status_info['agent_status'] = agent_status
                except:
                    pass

            # Add hosted agent info
            if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
                hosted_info = self._hosted_agents[agent_id]
                status_info.update({
                    'host': hosted_info.get('host'),
                    'port': hosted_info.get('port'),
                    'access': hosted_info.get('access'),
                    'public_name': hosted_info.get('public_name'),
                    'description': hosted_info.get('description')
                })

            # Add connection info
            connection_count = 0
            if hasattr(self, '_agent_connections') and agent_id in self._agent_connections:
                connection_count = len(self._agent_connections[agent_id])

            status_info['active_connections'] = connection_count

            return Result.json(data=status_info)

        except Exception as e:
            return Result.default_internal_error(
                info=f"Status retrieval failed: {str(e)}",
                exec_code=500
            )
cleanup_tools_interfaces() async

Cleanup all ToolsInterface instances.

Source code in toolboxv2/mods/isaa/module.py
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
async def cleanup_tools_interfaces(self):
    """
    Cleanup all ToolsInterface instances.
    """
    if not hasattr(self, 'tools_interfaces'):
        return

    async def cleanup_async():
        for name, tools_interface in self.tools_interfaces.items():
            if tools_interface:
                try:
                    await tools_interface.__aexit__(None, None, None)
                except Exception as e:
                    self.print(f"Error cleaning up ToolsInterface for {name}: {e}")

    # Run cleanup
    try:
        await cleanup_async()
        self.tools_interfaces.clear()
        self.print("Cleaned up all ToolsInterface instances")
    except Exception as e:
        self.print(f"Error during ToolsInterface cleanup: {e}")
configure_tools_interface(agent_name, **kwargs) async

Configure the ToolsInterface for a specific agent.

Parameters:

Name Type Description Default
agent_name str

Name of the agent

required
**kwargs

Configuration parameters

{}

Returns:

Type Description
bool

True if successful, False otherwise

Source code in toolboxv2/mods/isaa/module.py
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
async def configure_tools_interface(self, agent_name: str, **kwargs) -> bool:
    """
    Configure the ToolsInterface for a specific agent.

    Args:
        agent_name: Name of the agent
        **kwargs: Configuration parameters

    Returns:
        True if successful, False otherwise
    """
    tools_interface = self.get_tools_interface(agent_name)
    if not tools_interface:
        self.print(f"No ToolsInterface found for agent {agent_name}")
        return False

    try:
        # Configure based on provided parameters
        if 'base_directory' in kwargs:
            await tools_interface.set_base_directory(kwargs['base_directory'])

        if 'current_file' in kwargs:
            await tools_interface.set_current_file(kwargs['current_file'])

        if 'variables' in kwargs:
            tools_interface.ipython.user_ns.update(kwargs['variables'])

        self.print(f"Configured ToolsInterface for agent {agent_name}")
        return True

    except Exception as e:
        self.print(f"Failed to configure ToolsInterface for {agent_name}: {e}")
        return False
get_tools_interface(agent_name='self')

Get the ToolsInterface instance for a specific agent.

Parameters:

Name Type Description Default
agent_name str

Name of the agent

'self'

Returns:

Type Description
ToolsInterface | None

ToolsInterface instance or None if not found

Source code in toolboxv2/mods/isaa/module.py
859
860
861
862
863
864
865
866
867
868
869
870
871
872
def get_tools_interface(self, agent_name: str = "self") -> ToolsInterface | None:
    """
    Get the ToolsInterface instance for a specific agent.

    Args:
        agent_name: Name of the agent

    Returns:
        ToolsInterface instance or None if not found
    """
    if not hasattr(self, 'tools_interfaces'):
        return None

    return self.tools_interfaces.get(agent_name)
host_agent_ui(agent, host='0.0.0.0', port=None, access='local', registry_server=None, public_name=None, description=None, use_builtin_server=None) async

Unified agent hosting with WebSocket-enabled UI and optional registry publishing.

Parameters:

Name Type Description Default
agent

Agent or Chain instance to host

required
host str

Host address (default: 0.0.0.0 for remote access)

'0.0.0.0'
port int | None

Port number (auto-assigned if None)

None
access str

'local', 'remote', or 'registry'

'local'
registry_server str | None

Registry server URL for publishing (e.g., "ws://localhost:8080/ws/registry/connect")

None
public_name str | None

Public name for registry publishing

None
description str | None

Description for registry publishing

None
use_builtin_server bool

Use toolbox built-in server vs standalone Python server

None

Returns:

Type Description
dict[str, str]

Dictionary with access URLs and configuration

Source code in toolboxv2/mods/isaa/module.py
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
async def host_agent_ui(
    self,
    agent,
    host: str = "0.0.0.0",
    port: int | None = None,
    access: str = 'local',
    registry_server: str | None = None,
    public_name: str | None = None,
    description: str | None = None,
    use_builtin_server: bool = None
) -> dict[str, str]:
    """
    Unified agent hosting with WebSocket-enabled UI and optional registry publishing.

    Args:
        agent: Agent or Chain instance to host
        host: Host address (default: 0.0.0.0 for remote access)
        port: Port number (auto-assigned if None)
        access: 'local', 'remote', or 'registry'
        registry_server: Registry server URL for publishing (e.g., "ws://localhost:8080/ws/registry/connect")
        public_name: Public name for registry publishing
        description: Description for registry publishing
        use_builtin_server: Use toolbox built-in server vs standalone Python server

    Returns:
        Dictionary with access URLs and configuration
    """
    use_builtin_server = use_builtin_server or self.app.is_server
    if not hasattr(self, '_hosted_agents'):
        self._hosted_agents = {}

    agent_id = f"agent_{secrets.token_urlsafe(8)}"

    # Generate unique port if not specified
    if not port:
        port = 8765 + len(self._hosted_agents)

    # Store agent reference
    self._hosted_agents[agent_id] = {
        'agent': agent,
        'port': port,
        'host': host,
        'access': access,
        'public_name': public_name or f"Agent_{agent_id}",
        'description': description
    }

    result = {
        'agent_id': agent_id,
        'local_url': f"http://{host}:{port}",
        'status': 'starting'
    }

    if use_builtin_server:
        # Use toolbox built-in server
        result.update(await self._setup_builtin_server_hosting(agent_id, agent, host, port))
    else:
        # Use standalone Python server
        result.update(await self._setup_standalone_server_hosting(agent_id, agent, host, port))

    # Handle registry publishing if requested
    if access in ['remote', 'registry'] and registry_server:
        if not public_name:
            raise ValueError("public_name required for registry publishing")

        registry_result = await self._publish_to_registry(
            agent=agent,
            public_name=public_name,
            registry_server=registry_server,
            description=description,
            agent_id=agent_id
        )
        result.update(registry_result)

    self.app.print(f"🚀 Agent '{result.get('public_name', agent_id)}' hosted successfully!")
    self.app.print(f"   Local UI: {result['local_url']}")
    if 'public_url' in result:
        self.app.print(f"   Public URL: {result['public_url']}")
        self.app.print(f"   API Key: {result.get('api_key', 'N/A')}")

    return result
init_from_augment(augment, agent_name='self') async

Initialize from augmented data using new builder system

Source code in toolboxv2/mods/isaa/module.py
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
async def init_from_augment(self, augment, agent_name: str = 'self'):
    """Initialize from augmented data using new builder system"""

    # Handle agent_name parameter
    if isinstance(agent_name, str):
        pass  # Use string name
    elif hasattr(agent_name, 'config'):  # FlowAgentBuilder
        agent_name = agent_name.config.name
    else:
        raise ValueError(f"Invalid agent_name type: {type(agent_name)}")

    a_keys = augment.keys()

    # Load agent configurations
    if "Agents" in a_keys:
        agents_configs_dict = augment['Agents']
        self.deserialize_all(agents_configs_dict)
        self.print("Agent configurations loaded.")

    # Tools are now handled by the builder system during agent creation
    if "tools" in a_keys:
        self.print("Tool configurations noted - will be applied during agent building")
list_hosted_agents() async

List all currently hosted agents.

Source code in toolboxv2/mods/isaa/module.py
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
async def list_hosted_agents(self) -> dict[str, Any]:
    """List all currently hosted agents."""

    hosted_info = {
        'builtin_agents': {},
        'standalone_agents': {},
        'total_count': 0
    }

    # Built-in server agents
    if hasattr(self, '_hosted_agents'):
        for agent_id, info in self._hosted_agents.items():
            hosted_info['builtin_agents'][agent_id] = {
                'public_name': info.get('public_name'),
                'host': info.get('host'),
                'port': info.get('port'),
                'access': info.get('access'),
                'description': info.get('description')
            }

    # Standalone server agents
    if hasattr(self, '_standalone_servers'):
        for port, info in self._standalone_servers.items():
            hosted_info['standalone_agents'][info['agent_id']] = {
                'port': port,
                'thread_alive': info['thread'].is_alive(),
                'server_type': 'standalone'
            }

    hosted_info['total_count'] = len(hosted_info['builtin_agents']) + len(hosted_info['standalone_agents'])

    return hosted_info
publish_and_host_agent(agent, public_name, registry_server='ws://localhost:8080/ws/registry/connect', description=None, access_level='public') async

FIXED: Mit Debug-Ausgaben für Troubleshooting.

Source code in toolboxv2/mods/isaa/module.py
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
async def publish_and_host_agent(
    self,
    agent,
    public_name: str,
    registry_server: str = "ws://localhost:8080/ws/registry/connect",
    description: str | None = None,
    access_level: str = "public"
) -> dict[str, Any]:
    """FIXED: Mit Debug-Ausgaben für Troubleshooting."""

    if hasattr(agent, 'name') and not hasattr(agent, 'amd') and hasattr(agent, 'a_run'):
        agent.amd = lambda :None
        agent.amd.name = agent.name

    try:
        # Registry Client initialisieren
        from toolboxv2.mods.registry.client import get_registry_client
        registry_client = get_registry_client(self.app)

        self.app.print(f"Connecting to registry server: {registry_server}")
        await registry_client.connect(registry_server)

        # Progress Callback für Live-Updates einrichten
        callback_success = await self.setup_live_progress_callback(agent, registry_client, f"agent_{agent.amd.name}")
        if not callback_success:
            self.app.print("Warning: Progress callback setup failed")
        else:
            self.app.print("✅ Progress callback setup successful")

        # Agent beim Registry registrieren
        self.app.print(f"Registering agent: {public_name}")
        registration_info = await registry_client.register(
            agent_instance=agent,
            public_name=public_name,
            description=description or f"Agent: {public_name}"
        )

        if not registration_info:
            return {"error": "Registration failed", "success": False}

        self.app.print(f"✅ Agent registration successful: {registration_info.public_agent_id}")

        result = {
            "success": True,
            "agent_name": public_name,
            "public_agent_id": registration_info.public_agent_id,
            "public_api_key": registration_info.public_api_key,
            "public_url": registration_info.public_url,
            "registry_server": registry_server,
            "access_level": access_level,
            "ui_url": registration_info.public_url.replace("/api/registry/run", "/api/registry/ui"),
            "websocket_url": registry_server.replace("/connect", "/ui_connect"),
            "status": "registered"
        }

        return result

    except Exception as e:
        self.app.print(f"Failed to publish agent: {e}")
        return {"error": str(e), "success": False}
setup_live_progress_callback(agent, registry_client, agent_id=None) async

Enhanced setup for live progress callback with proper error handling.

Source code in toolboxv2/mods/isaa/module.py
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
async def setup_live_progress_callback(self, agent, registry_client, agent_id: str = None):
    """Enhanced setup for live progress callback with proper error handling."""

    if not registry_client:
        self.app.print("Warning: No registry client provided for progress updates")
        return False

    if not registry_client.is_connected:
        self.app.print("Warning: Registry client is not connected")
        return False

    progress_tracker = EnhancedProgressTracker()

    # Generate agent ID if not provided
    if not agent_id:
        agent_id = getattr(agent, 'name', f'agent_{id(agent)}')

    async def enhanced_live_progress_callback(event: ProgressEvent):
        """Enhanced progress callback with comprehensive data extraction."""
        try:
            # Validate event
            if not event:
                self.app.print("Warning: Received null progress event")
                return

            # Debug output for local development
            event_type = getattr(event, 'event_type', 'unknown')
            status = getattr(event, 'status', 'unknown')
            agent_name = getattr(event, 'agent_name', 'Unknown Agent')

            self.app.print(f"📊 Progress Event: {event_type} | {status} | {agent_name}")

            # Extract comprehensive progress data
            progress_data = progress_tracker.extract_progress_data(event)

            # Prepare enhanced progress message
            ui_progress_data = {
                "agent_id": agent_id,
                "event_type": event_type,
                "status": status.value if hasattr(status, 'value') else str(status),
                "timestamp": getattr(event, 'timestamp', asyncio.get_event_loop().time()),
                "agent_name": agent_name,
                "node_name": getattr(event, 'node_name', 'Unknown'),
                "session_id": getattr(event, 'session_id', None),

                # Core event metadata
                "metadata": {
                    **getattr(event, 'metadata', {}),
                    "event_id": getattr(event, 'event_id', f"evt_{asyncio.get_event_loop().time()}"),
                    "sequence_number": getattr(event, 'sequence_number', 0),
                    "parent_event_id": getattr(event, 'parent_event_id', None)
                },

                # Detailed progress data for UI panels
                "progress_data": progress_data,

                # UI-specific flags for selective updates
                "ui_flags": {
                    "should_update_outline": bool(progress_data.get('outline')),
                    "should_update_activity": bool(progress_data.get('activity')),
                    "should_update_meta_tools": bool(progress_data.get('meta_tool')),
                    "should_update_system": bool(progress_data.get('system')),
                    "should_update_graph": bool(progress_data.get('graph')),
                    "is_error": event_type.lower() in ['error', 'exception', 'failed'],
                    "is_completion": event_type.lower() in ['complete', 'finished', 'success'],
                    "requires_user_input": getattr(event, 'requires_user_input', False)
                },

                # Performance metrics
                "performance": {
                    "execution_time": getattr(event, 'execution_time', None),
                    "memory_delta": getattr(event, 'memory_delta', None),
                    "tokens_used": getattr(event, 'tokens_used', None),
                    "api_calls_made": getattr(event, 'api_calls_made', None)
                }
            }

            # Send live update to registry server
            await registry_client.send_ui_progress(ui_progress_data)

            # Also send agent status update if this is a significant event
            if event_type in ['started', 'completed', 'error', 'paused', 'resumed']:
                agent_status = 'processing'
                if event_type == 'completed':
                    agent_status = 'idle'
                elif event_type == 'error':
                    agent_status = 'error'
                elif event_type == 'paused':
                    agent_status = 'paused'

                await registry_client.send_agent_status(
                    agent_id=agent_id,
                    status=agent_status,
                    details={
                        "last_event": event_type,
                        "last_update": ui_progress_data["timestamp"],
                        "current_node": progress_data.get('graph', {}).get('current_node', 'Unknown')
                    }
                )

            # Log successful progress update
            self.app.print(f"✅ Sent progress update: {event_type} -> Registry Server")

        except Exception as e:
            self.app.print(f"❌ Progress callback error: {e}")
            # Send error notification to UI
            try:
                await registry_client.send_ui_progress({
                    "agent_id": agent_id,
                    "event_type": "progress_callback_error",
                    "status": "error",
                    "timestamp": asyncio.get_event_loop().time(),
                    "agent_name": getattr(agent, 'name', 'Unknown'),
                    "metadata": {"error": str(e)},
                    "ui_flags": {"is_error": True}
                })
            except Exception as nested_error:
                self.app.print(f"Failed to send error notification: {nested_error}")

    # Set up progress callback with enhanced error handling
    callback_set = False

    if hasattr(agent, 'set_progress_callback'):
        try:
            self.app.print(f"🔧 Setting progress callback via set_progress_callback for agent: {agent_id}")
            agent.set_progress_callback(enhanced_live_progress_callback)
            callback_set = True
        except Exception as e:
            self.app.print(f"Failed to set progress callback via set_progress_callback: {e}")

    if not callback_set and hasattr(agent, 'progress_callback'):
        try:
            self.app.print(f"🔧 Setting progress callback via direct assignment for agent: {agent_id}")
            agent.progress_callback = enhanced_live_progress_callback
            callback_set = True
        except Exception as e:
            self.app.print(f"Failed to set progress callback via direct assignment: {e}")

    if not callback_set:
        self.app.print(f"⚠️ Warning: Agent {agent_id} doesn't support progress callbacks")
        return False

    # Send initial agent status
    try:
        await registry_client.send_agent_status(
            agent_id=agent_id,
            status='online',
            details={
                "progress_callback_enabled": True,
                "callback_setup_time": asyncio.get_event_loop().time(),
                "agent_type": type(agent).__name__
            }
        )
        self.app.print(f"✅ Progress callback successfully set up for agent: {agent_id}")
    except Exception as e:
        self.app.print(f"Failed to send initial agent status: {e}")

    return True
stop_hosted_agent(agent_id=None, port=None) async

Stop a hosted agent by agent_id or port.

Source code in toolboxv2/mods/isaa/module.py
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
async def stop_hosted_agent(self, agent_id: str = None, port: int = None):
    """Stop a hosted agent by agent_id or port."""

    if not hasattr(self, '_hosted_agents') and not hasattr(self, '_standalone_servers'):
        self.app.print("No hosted agents found")
        return False

    # Stop by agent_id
    if agent_id:
        if hasattr(self, '_hosted_agents') and agent_id in self._hosted_agents:
            agent_info = self._hosted_agents[agent_id]
            agent_port = agent_info.get('port')

            # Stop standalone server if exists
            if hasattr(self, '_standalone_servers') and agent_port in self._standalone_servers:
                server_info = self._standalone_servers[agent_port]
                try:
                    server_info['server'].shutdown()
                    self.app.print(f"Stopped standalone server for agent {agent_id}")
                except:
                    pass

            # Clean up hosted agent info
            del self._hosted_agents[agent_id]
            self.app.print(f"Stopped hosted agent {agent_id}")
            return True

    # Stop by port
    if port:
        if hasattr(self, '_standalone_servers') and port in self._standalone_servers:
            server_info = self._standalone_servers[port]
            try:
                server_info['server'].shutdown()
                self.app.print(f"Stopped server on port {port}")
                return True
            except Exception as e:
                self.app.print(f"Failed to stop server on port {port}: {e}")
                return False

    self.app.print("Agent or port not found")
    return False

ui

get_agent_ui_html()

Produktionsfertige UI mit Live-Progress-Tracking.

Source code in toolboxv2/mods/isaa/ui.py
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
def get_agent_ui_html() -> str:
    """Produktionsfertige UI mit Live-Progress-Tracking."""

    return """<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Agent Registry - Live Interface</title>
    <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
    <style>
        /* Modernes Dark Theme UI */
        :root {
            --bg-primary: #0d1117;
            --bg-secondary: #161b22;
            --bg-tertiary: #21262d;
            --text-primary: #f0f6fc;
            --text-secondary: #8b949e;
            --text-muted: #6e7681;
            --accent-blue: #58a6ff;
            --accent-green: #3fb950;
            --accent-red: #f85149;
            --accent-orange: #d29922;
            --accent-purple: #a5a5f5;
            --accent-cyan: #39d0d8;
            --border-color: #30363d;
            --shadow: 0 2px 8px rgba(0, 0, 0, 0.3);

            --sidebar-width: 300px;
            --progress-width: 660px;
            --sidebar-collapsed: 60px;
            --progress-collapsed: 60px;
        }
        /* Enhanced Progress Panel Styles */
        .progress-section {
            margin-bottom: 16px;
        }

        /* ADD to existing CSS */
        .event-status-badge {
            padding: 2px 6px;
            border-radius: 3px;
            font-size: 10px;
            font-weight: 500;
        }

        .event-status-badge.completed {
            background: var(--accent-green);
            color: white;
        }

        .event-status-badge.running {
            background: var(--accent-orange);
            color: white;
        }

        .event-status-badge.failed, .event-status-badge.error {
            background: var(--accent-red);
            color: white;
        }

        .event-status-badge.starting {
            background: var(--accent-cyan);
            color: white;
        }

        .progress-item.expandable[data-event-id*="tool_call"] {
            border-left-color: var(--accent-orange);
        }

        .progress-item.expandable[data-event-id*="llm_call"] {
            border-left-color: var(--accent-purple);
        }

        .progress-item.expandable[data-event-id*="meta_tool"] {
            border-left-color: var(--accent-cyan);
        }

        .progress-item.expandable[data-event-id*="error"] {
            border-left-color: var(--accent-red);
            background: rgba(248, 81, 73, 0.02);
        }

        .section-title.expandable-section {
            cursor: pointer;
            display: flex;
            justify-content: space-between;
            align-items: center;
            padding: 8px 12px;
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 6px;
            transition: all 0.2s;
        }

        .section-title.expandable-section:hover {
            background: var(--bg-tertiary);
        }

        .section-toggle {
            transition: transform 0.2s;
            font-size: 12px;
        }

        .section-content {
            max-height: 0;
            overflow: hidden;
            transition: max-height 0.3s ease-out;
            background: var(--bg-primary);
            border-radius: 0 0 6px 6px;
        }

        .section-content.expanded {
            max-height: 900px;
            padding: 12px;
            border: 1px solid var(--border-color);
            border-top: none;
            overflow-y: auto;
        }

        .no-data {
            color: var(--text-muted);
            font-size: 12px;
            text-align: center;
            padding: 12px;
            font-style: italic;
        }

        /* Expandable Progress Items */
        .progress-item.expandable {
            cursor: pointer;
            transition: all 0.2s;
            margin-bottom: 8px;
        }

        .progress-item.expandable:hover {
            transform: translateY(-1px);
            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15);
        }

        .progress-item.expandable.expanded {
            border-color: var(--accent-blue);
        }

        .progress-item.expandable.latest {
            border-left: 3px solid var(--accent-green);
            background: rgba(63, 185, 80, 0.05);
        }

        .progress-item-header {
            display: flex;
            align-items: center;
            gap: 8px;
            padding: 8px 12px;
        }

        .progress-meta {
            margin-left: auto;
            display: flex;
            align-items: center;
            gap: 8px;
        }

        .expand-indicator {
            transition: transform 0.2s;
            font-size: 12px;
            color: var(--text-muted);
        }

        .progress-item.expanded .expand-indicator {
            transform: rotate(180deg);
        }

        .progress-summary {
            padding: 0 12px 8px 36px;
            font-size: 11px;
            color: var(--text-secondary);
        }

        .progress-item-expanded {
            max-height: 0;
            overflow: hidden;
            transition: max-height 0.3s ease-out;
            background: var(--bg-secondary);
            border-top: 1px solid var(--border-color);
        }

        .progress-item-expanded.active {
            max-height: 400px;
            padding: 12px;
            overflow-y: auto;
        }

        .expanded-section {
            margin-bottom: 12px;
        }

        .expanded-section-title {
            font-size: 12px;
            font-weight: 600;
            color: var(--accent-blue);
            margin-bottom: 6px;
            padding-bottom: 4px;
            border-bottom: 1px solid var(--border-color);
        }

        .event-field {
            display: flex;
            justify-content: space-between;
            align-items: flex-start;
            padding: 4px 0;
            font-size: 11px;
        }

        .event-field-label {
            font-weight: 500;
            color: var(--text-secondary);
            min-width: 80px;
        }

        .event-field-value {
            color: var(--text-primary);
            text-align: right;
            flex: 1;
        }

        .event-field-value.json {
            background: var(--bg-primary);
            border-radius: 4px;
            padding: 6px;
            font-family: monospace;
            font-size: 10px;
            text-align: left;
            white-space: pre-wrap;
            max-height: 100px;
            overflow-y: auto;
        }

        /* ADD to existing CSS */
.thinking-step.outline-step {
    border-color: var(--accent-cyan);
    background: rgba(57, 208, 216, 0.05);
}

.thinking-step.outline-step.completed {
    border-color: var(--accent-green);
    background: rgba(63, 185, 80, 0.05);
}

.thinking-step.outline-step.running {
    border-color: var(--accent-orange);
    background: rgba(210, 153, 34, 0.05);
}

.outline-progress {
    margin: 8px 0;
}

.progress-info {
    display: flex;
    justify-content: space-between;
    align-items: center;
    margin-bottom: 6px;
}

.progress-text {
    font-size: 11px;
    color: var(--text-secondary);
    font-weight: 500;
}

.progress-percentage {
    font-size: 11px;
    color: var(--accent-blue);
    font-weight: 600;
}

.progress-bar-container {
    margin-bottom: 8px;
}

.progress-bar {
    height: 6px;
    background: var(--bg-primary);
    border-radius: 3px;
    overflow: hidden;
    border: 1px solid var(--border-color);
}

.progress-bar-fill {
    height: 100%;
    background: linear-gradient(90deg, var(--accent-cyan), var(--accent-blue));
    transition: width 0.5s ease-out;
    position: relative;
}

.progress-bar-fill::after {
    content: '';
    position: absolute;
    top: 0;
    left: 0;
    right: 0;
    bottom: 0;
    background: linear-gradient(90deg, transparent, rgba(255,255,255,0.2), transparent);
    animation: shimmer 2s infinite;
}

@keyframes shimmer {
    0% { transform: translateX(-100%); }
    100% { transform: translateX(100%); }
}

.step-completed {
    color: var(--accent-green);
    font-size: 10px;
    text-align: center;
    font-weight: 500;
}

.step-working {
    color: var(--accent-orange);
    font-size: 10px;
    text-align: center;
    font-style: italic;
}

.context-info {
    display: flex;
    justify-content: center;
    gap: 12px;
    margin-top: 8px;
    padding-top: 8px;
    border-top: 1px solid var(--border-color);
}

.context-item {
    font-size: 10px;
    color: var(--text-muted);
    background: var(--bg-primary);
    padding: 2px 6px;
    border-radius: 3px;
    border: 1px solid var(--border-color);
}

.thinking-step.plan-created {
    border-color: var(--accent-blue);
    background: rgba(88, 166, 255, 0.05);
}

.plan-details {
    text-align: center;
}

.plan-info {
    display: flex;
    justify-content: center;
    gap: 12px;
    margin-bottom: 8px;
}

.plan-item {
    font-size: 11px;
    color: var(--text-secondary);
    background: var(--bg-primary);
    padding: 4px 8px;
    border-radius: 4px;
    border: 1px solid var(--border-color);
}

.plan-ready, .outline-ready {
    margin-top: 8px;
    color: var(--accent-green);
    font-size: 10px;
    text-align: center;
}

.step-status {
    padding: 2px 6px;
    border-radius: 3px;
    font-size: 10px;
    font-weight: 500;
    margin-left: auto;
}

.step-status.completed {
    background: var(--accent-green);
    color: white;
}

.step-status.running {
    background: var(--accent-orange);
    color: white;
}

.step-status.ready {
    background: var(--accent-blue);
    color: white;
}

        /* Enhanced Chat Integration Styles */
        .thinking-step {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 10px 12px;
            margin: 8px 0;
            font-size: 13px;
            transition: all 0.2s;
        }

        .thinking-step:hover {
            transform: translateY(-1px);
            box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
        }

        .thinking-step.reasoning-loop {
            border-color: var(--accent-purple);
            background: rgba(165, 165, 245, 0.05);
        }

        .thinking-step.outline-created {
            border-color: var(--accent-cyan);
            background: rgba(57, 208, 216, 0.05);
        }

        .thinking-step.task-progress.starting {
            border-color: var(--accent-orange);
            background: rgba(210, 153, 34, 0.05);
        }

        .thinking-step.task-progress.completed {
            border-color: var(--accent-green);
            background: rgba(63, 185, 80, 0.05);
        }

        .thinking-step.task-progress.error {
            border-color: var(--accent-red);
            background: rgba(248, 81, 73, 0.05);
        }

        .thinking-step-header {
            display: flex;
            align-items: center;
            gap: 8px;
            font-weight: 600;
            margin-bottom: 8px;
            color: var(--text-primary);
        }

        .step-progress, .step-info, .step-status {
            margin-left: auto;
            font-size: 10px;
            font-weight: normal;
            color: var(--text-muted);
            background: var(--bg-primary);
            padding: 2px 6px;
            border-radius: 3px;
        }

        .priority-badge {
            padding: 2px 6px;
            border-radius: 3px;
            font-size: 10px;
            font-weight: 500;
        }

        .priority-badge.high {
            background: var(--accent-red);
            color: white;
        }

        .priority-badge.normal {
            background: var(--accent-blue);
            color: white;
        }

        .priority-badge.low {
            background: var(--text-muted);
            color: white;
        }

        /* Performance Metrics Grid */
        .metrics-grid {
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(100px, 1fr));
            gap: 8px;
        }

        .metric-card {
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 6px;
            padding: 8px;
            text-align: center;
        }

        .metric-label {
            font-size: 10px;
            color: var(--text-muted);
            margin-bottom: 4px;
        }

        .metric-value {
            font-size: 14px;
            font-weight: 600;
            color: var(--accent-blue);
        }

        .reasoning-metrics .metric-grid {
            display: grid;
            grid-template-columns: repeat(3, 1fr);
            gap: 8px;
            margin-bottom: 8px;
        }

        .metric-item {
            display: flex;
            justify-content: space-between;
            align-items: center;
            font-size: 11px;
        }

        .metric-label {
            color: var(--text-muted);
        }

        .metric-value {
            color: var(--text-primary);
            font-weight: 500;
        }

        /* Progress Bar */
        .progress-bar-container {
            margin: 8px 0;
        }

        .progress-bar-info {
            display: flex;
            justify-content: space-between;
            font-size: 10px;
            color: var(--text-muted);
            margin-bottom: 4px;
        }

        .progress-bar {
            height: 4px;
            background: var(--bg-primary);
            border-radius: 2px;
            overflow: hidden;
        }

        .progress-bar-fill {
            height: 100%;
            background: var(--accent-blue);
            transition: width 0.5s ease-out;
        }

        /* Outline Display */
        .outline-steps {
            margin: 8px 0;
        }

        .outline-step {
            display: flex;
            align-items: flex-start;
            gap: 8px;
            margin-bottom: 4px;
            font-size: 11px;
        }

        .step-number {
            color: var(--accent-blue);
            font-weight: 600;
            min-width: 20px;
        }

        .step-text {
            color: var(--text-primary);
            line-height: 1.3;
        }

        .context-metrics {
            display: grid;
            grid-template-columns: repeat(3, 1fr);
            gap: 8px;
            margin-bottom: 8px;
        }

        .context-metric {
            display: flex;
            flex-direction: column;
            align-items: center;
            font-size: 10px;
            padding: 6px;
            background: var(--bg-primary);
            border-radius: 4px;
        }

        .context-label {
            color: var(--text-muted);
            margin-bottom: 2px;
        }

        .context-value {
            color: var(--text-primary);
            font-weight: 600;
        }

        .task-description {
            margin-bottom: 6px;
            font-weight: 500;
        }

        .task-timing {
            font-size: 10px;
            color: var(--accent-green);
        }

        .task-error {
            font-size: 10px;
            color: var(--accent-red);
            background: rgba(248, 81, 73, 0.1);
            padding: 4px;
            border-radius: 3px;
            margin-top: 4px;
        }

        .reasoning-insight {
            margin-top: 8px;
            font-size: 11px;
            color: var(--accent-purple);
            text-align: center;
            font-style: italic;
        }

        .idle-status {
            border-color: var(--accent-green);
            background: rgba(63, 185, 80, 0.02);
        }

        @media (max-width: 1200px) {
            :root {
                --sidebar-width: 250px;
                --progress-width: 580px;
            }
        }

        @media (max-width: 1024px) {
            :root {
                --sidebar-width: 220px;
                --progress-width: 460px;
            }
        }

        .sidebar.collapsed::before {
            content: '📋';
            font-size: 20px;
            display: flex;
            align-items: center;
            justify-content: center;
            padding: 20px 0;
            border-bottom: 1px solid var(--border-color);
        }

        .progress-panel.collapsed::before {
            content: '📊';
            font-size: 20px;
            display: flex;
            align-items: center;
            justify-content: center;
            padding: 20px 0;
            border-bottom: 1px solid var(--border-color);
            writing-mode: vertical-lr;
        }

        .sidebar, .progress-panel {
            transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1);
        }

        .main-container {
            transition: grid-template-columns 0.3s cubic-bezier(0.4, 0, 0.2, 1);
        }

        * { margin: 0; padding: 0; box-sizing: border-box; }

        body {
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
            background: var(--bg-primary);
            color: var(--text-primary);
            height: 100vh;
            display: flex;
            flex-direction: column;
            overflow: hidden;
        }

        html, body {
            height: 100%;
            overflow: hidden;
        }

        .api-key-modal {
            position: fixed;
            top: 0;
            left: 0;
            right: 0;
            bottom: 0;
            background: rgba(0, 0, 0, 0.8);
            display: flex;
            align-items: center;
            justify-content: center;
            z-index: 1000;
        }

        .api-key-content {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            border-radius: 12px;
            padding: 24px;
            max-width: 500px;
            width: 90%;
            text-align: center;
        }

        .api-key-title {
            font-size: 20px;
            font-weight: 600;
            color: var(--accent-blue);
            margin-bottom: 16px;
        }

        .api-key-description {
            color: var(--text-secondary);
            margin-bottom: 20px;
            line-height: 1.5;
        }

        .api-key-input {
            width: 100%;
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 12px;
            color: var(--text-primary);
            font-size: 14px;
            margin-bottom: 16px;
        }

        .api-key-button {
            background: var(--accent-blue);
            color: white;
            border: none;
            border-radius: 8px;
            padding: 12px 24px;
            cursor: pointer;
            font-weight: 600;
        }

        /* Updated Header */
        .header {
            background: var(--bg-tertiary);
            padding: 16px 24px;
            border-bottom: 1px solid var(--border-color);
            display: flex;
            align-items: center;
            justify-content: space-between;
            box-shadow: var(--shadow);
            flex-shrink: 0;
        }

        .header-controls {
            display: flex;
            align-items: center;
            gap: 12px;
        }

        .panel-toggle {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            color: var(--text-primary);
            padding: 8px 12px;
            border-radius: 6px;
            cursor: pointer;
            font-size: 12px;
            transition: all 0.2s;
        }

        .panel-toggle:hover {
            background: var(--bg-primary);
        }

        .panel-toggle.active {
            background: var(--accent-blue);
            color: white;
        }

        .logo {
            display: flex;
            align-items: center;
            gap: 12px;
            font-size: 20px;
            font-weight: 700;
            color: var(--accent-blue);
        }

        .connection-status {
            display: flex;
            align-items: center;
            gap: 12px;
        }

        .status-indicator {
            display: flex;
            align-items: center;
            gap: 8px;
            padding: 8px 12px;
            border-radius: 6px;
            font-size: 14px;
            font-weight: 500;
        }

        .status-indicator.connected {
            background: rgba(63, 185, 80, 0.1);
            color: var(--accent-green);
            border: 1px solid var(--accent-green);
        }

        .status-indicator.disconnected {
            background: rgba(248, 81, 73, 0.1);
            color: var(--accent-red);
            border: 1px solid var(--accent-red);
        }

        .status-dot {
            width: 8px;
            height: 8px;
            border-radius: 50%;
            background: currentColor;
            animation: pulse 2s infinite;
        }

        .status-dot.connected { animation: none; }

        @keyframes pulse {
            0%, 100% { opacity: 1; }
            50% { opacity: 0.4; }
        }

        /* FIXED: Better grid layout that properly handles collapsing */
        .main-container {
            display: grid;
            grid-template-areas: "sidebar chat progress";
            grid-template-columns: var(--sidebar-width) 1fr var(--progress-width);
            flex: 1;
            overflow: hidden;
            min-height: 0;
            height: 100%;
        }

        .main-container.sidebar-collapsed {
            grid-template-columns: var(--sidebar-collapsed) 1fr var(--progress-width);
        }

        .main-container.progress-collapsed {
            grid-template-columns: var(--sidebar-width) 1fr var(--progress-collapsed);
        }

        .main-container.both-collapsed {
            grid-template-columns: var(--sidebar-collapsed) 1fr var(--progress-collapsed);
        }

        .sidebar {
            grid-area: sidebar;
            background: var(--bg-secondary);
            border-right: 1px solid var(--border-color);
            display: flex;
            flex-direction: column;
            overflow: hidden;
            height: 100%;
        }

        .sidebar.collapsed .agents-list,
        .sidebar.collapsed .system-info {
            display: none;
        }

        .sidebar.collapsed .sidebar-header {
            padding: 12px 8px;
            justify-content: center;
        }

        .sidebar.collapsed .sidebar-title {
            display: none;
        }

        .sidebar.collapsed .collapse-btn {
            writing-mode: vertical-lr;
            text-orientation: mixed;
        }

        .progress-panel.collapsed .collapse-btn {
            writing-mode: vertical-lr;
            text-orientation: mixed;
            transform: rotate(180deg);
        }

        .sidebar-header {
            padding: 12px 16px;
            background: var(--bg-tertiary);
            border-bottom: 1px solid var(--border-color);
            display: flex;
            align-items: center;
            justify-content: space-between;
            min-height: 48px;
        }

        .sidebar-title {
            font-size: 14px;
            font-weight: 600;
            color: var(--text-secondary);
            text-transform: uppercase;
        }

        .collapse-btn {
            background: none;
            border: none;
            color: var(--text-muted);
            cursor: pointer;
            padding: 4px;
            border-radius: 4px;
            transition: all 0.2s;
        }

        .collapse-btn:hover {
            background: var(--bg-primary);
            color: var(--text-primary);
        }

        /* FIXED: Chat area properly uses grid area and expands */
        .chat-area {
            grid-area: chat;
            display: flex;
            flex-direction: column;
            background: var(--bg-primary);
            min-height: 0;
            height: 100%;
            overflow: hidden;
        }

        /* Updated Progress Panel */
        .progress-panel {
            grid-area: progress;
            background: var(--bg-secondary);
            border-left: 1px solid var(--border-color);
            display: flex;
            flex-direction: column;
            overflow: hidden;
            height: 100%;
        }

        .progress-panel.collapsed .panel-content {
            display: none;
        }

        .progress-panel.collapsed .progress-header {
            padding: 12px 8px;
            justify-content: center;
            writing-mode: vertical-lr;
            text-orientation: mixed;
        }

        .progress-panel.collapsed .progress-header span {
            transform: rotate(180deg);
        }

        .progress-header {
            padding: 12px 16px;
            background: var(--bg-tertiary);
            border-bottom: 1px solid var(--border-color);
            display: flex;
            align-items: center;
            justify-content: space-between;
            font-weight: 600;
            font-size: 14px;
            min-height: 48px;
        }

        /* ADD to existing CSS */
        .progress-item.llm_call {
            border-left: 3px solid var(--accent-purple);
        }

        .progress-item.llm_call.latest {
            border-left: 3px solid var(--accent-purple);
            background: rgba(165, 165, 245, 0.03);
        }

        .progress-item.llm_call .progress-icon {
            color: var(--accent-purple);
        }

        .progress-summary {
            padding: 0 12px 8px 36px;
            font-size: 10px;
            color: var(--text-secondary);
            line-height: 1.3;
        }

        .event-field-value.json {
            background: var(--bg-primary);
            border-radius: 4px;
            padding: 8px;
            font-family: 'Consolas', 'Monaco', monospace;
            font-size: 10px;
            text-align: left;
            white-space: pre-wrap;
            max-height: 300px;
            overflow-y: auto;
            border: 1px solid var(--border-color);
            word-break: break-all;
        }

        .expanded-section {
            margin-bottom: 12px;
            border-bottom: 1px solid rgba(48, 54, 61, 0.3);
            padding-bottom: 8px;
        }

        .expanded-section:last-child {
            border-bottom: none;
            margin-bottom: 0;
        }

        /* FIXED: Hide mobile tabs on desktop by default */
        .mobile-tabs {
            display: none;
        }

        /* Mobile Responsive */
        @media (max-width: 768px) {
            .main-container {
                display: flex !important;
                flex-direction: column;
                height: 100%;
                grid-template-areas: none;
                grid-template-columns: none;
            }

            .mobile-tabs {
                display: flex;
                background: var(--bg-tertiary);
                border-bottom: 1px solid var(--border-color);
                flex-shrink: 0;
            }

            .header-controls {
                display: none;
            }

            .mobile-tab {
                flex: 1;
                padding: 12px;
                text-align: center;
                background: var(--bg-secondary);
                border-right: 1px solid var(--border-color);
                cursor: pointer;
                transition: all 0.2s;
                font-size: 14px;
            }

            .mobile-tab:last-child {
                border-right: none;
            }

            .mobile-tab.active {
                background: var(--accent-blue);
                color: white;
            }

            .sidebar,
            .progress-panel {
                flex: 1;
                border-right: none;
                border-left: none;
                border-bottom: 1px solid var(--border-color);
                min-height: 0;
                max-height: none;
            }

            .chat-area {
                flex: 1;
                min-height: 0;
            }

            .sidebar,
            .chat-area,
            .progress-panel {
                display: none;
            }
        }

        @media (min-width: 769px) {
            .main-container {
                display: grid !important;
            }

            .sidebar,
            .chat-area,
            .progress-panel {
                display: flex !important;
                height: 100%;
            }
        }

        .agents-list {
            flex: 1;
            overflow-y: auto;
            padding: 16px;
            min-height: 0;
        }

        .agents-header {
            font-size: 14px;
            font-weight: 600;
            color: var(--text-secondary);
            margin-bottom: 12px;
            text-transform: uppercase;
            letter-spacing: 0.5px;
        }

        .agent-item {
            padding: 12px;
            margin-bottom: 8px;
            background: var(--bg-tertiary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            cursor: pointer;
            transition: all 0.2s;
        }

        .agent-item:hover {
            border-color: var(--accent-blue);
            transform: translateY(-1px);
        }

        .agent-item.active {
            border-color: var(--accent-blue);
            background: rgba(88, 166, 255, 0.1);
        }

        .agent-name {
            font-weight: 600;
            color: var(--text-primary);
            margin-bottom: 4px;
        }

        .agent-description {
            font-size: 12px;
            color: var(--text-muted);
            margin-bottom: 6px;
        }

        .agent-status {
            display: flex;
            align-items: center;
            gap: 6px;
            font-size: 11px;
        }

        .agent-status.online { color: var(--accent-green); }
        .agent-status.offline { color: var(--accent-red); }

        .chat-header {
            padding: 16px 20px;
            border-bottom: 1px solid var(--border-color);
            background: var(--bg-tertiary);
            flex-shrink: 0;
        }

        .chat-title {
            font-size: 16px;
            font-weight: 600;
            color: var(--text-primary);
            margin-bottom: 4px;
        }

        .chat-subtitle {
            font-size: 12px;
            color: var(--text-muted);
        }

        .messages-container {
            flex: 1;
            overflow-y: auto;
            padding: 20px;
            display: flex;
            flex-direction: column;
            gap: 16px;
            min-height: 0;
        }

        .message {
            display: flex;
            gap: 12px;
            max-width: 85%;
        }

        .message.user {
            flex-direction: row-reverse;
            margin-left: auto;
        }

        .message-avatar {
            width: 36px;
            height: 36px;
            border-radius: 50%;
            display: flex;
            align-items: center;
            justify-content: center;
            font-size: 14px;
            font-weight: 600;
            flex-shrink: 0;
        }

        .message.user .message-avatar {
            background: var(--accent-blue);
            color: white;
        }

        .message.agent .message-avatar {
            background: var(--accent-green);
            color: white;
        }

        .message-content {
            padding: 12px 16px;
            border-radius: 16px;
            line-height: 1.5;
            font-size: 14px;
        }

        .message.user .message-content {
            background: var(--accent-blue);
            color: white;
        }

        .message.agent .message-content {
            background: var(--bg-tertiary);
            border: 1px solid var(--border-color);
            color: var(--text-primary);
        }

        /* NEW: Thinking step styles */
        .thinking-step {
            background: var(--bg-secondary);
            border: 1px solid var(--accent-purple);
            border-radius: 12px;
            padding: 12px 16px;
            margin: 8px 0;
            font-size: 13px;
            color: var(--text-secondary);
        }

        .thinking-step.outline-step {
            border-color: var(--accent-cyan);
            background: rgba(57, 208, 216, 0.05);
        }

        .thinking-step-header {
            display: flex;
            align-items: center;
            gap: 8px;
            font-weight: 600;
            margin-bottom: 6px;
            color: var(--text-primary);
        }

        .thinking-step-content {
            line-height: 1.4;
        }

        .message-input {
            border-top: 1px solid var(--border-color);
            padding: 16px 20px;
            display: flex;
            gap: 12px;
            flex-shrink: 0;
            background: var(--bg-secondary);
        }

        .input-field {
            flex: 1;
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 12px;
            color: var(--text-primary);
            font-size: 14px;
        }

        .input-field:focus {
            outline: none;
            border-color: var(--accent-blue);
        }

        .send-button {
            background: var(--accent-blue);
            color: white;
            border: none;
            border-radius: 8px;
            padding: 12px 20px;
            cursor: pointer;
            font-weight: 600;
            transition: all 0.2s;
        }

        .send-button:hover:not(:disabled) {
            background: #4493f8;
            transform: translateY(-1px);
        }

        .send-button:disabled {
            opacity: 0.5;
            cursor: not-allowed;
            transform: none;
        }

        .panel-header {
            padding: 16px;
            background: var(--bg-tertiary);
            border-bottom: 1px solid var(--border-color);
            font-weight: 600;
            font-size: 14px;
        }

        .panel-content {
            flex: 1;
            overflow-y: auto;
            padding: 16px;
            min-height: 0;
        }

        .progress-section {
            margin-bottom: 20px;
        }

        .section-title {
            font-size: 12px;
            font-weight: 600;
            color: var(--text-muted);
            text-transform: uppercase;
            margin-bottom: 8px;
            letter-spacing: 0.5px;
        }

        /* NEW: Enhanced progress item styles */
        .progress-item {
            background: var(--bg-primary);
            border: 1px solid var(--border-color);
            border-radius: 8px;
            padding: 12px;
            margin-bottom: 8px;
            font-size: 12px;
            transition: all 0.2s;
        }

        .progress-item:hover {
            border-color: var(--accent-blue);
            transform: translateY(-1px);
        }

        .progress-item-header {
            display: flex;
            align-items: center;
            gap: 8px;
            margin-bottom: 6px;
        }

        .progress-icon {
            width: 16px;
            text-align: center;
            font-size: 14px;
        }

        .progress-title {
            font-weight: 500;
            color: var(--text-primary);
            flex: 1;
        }

        .progress-status {
            padding: 2px 6px;
            border-radius: 3px;
            font-size: 10px;
            font-weight: 500;
        }

        .progress-status.running {
            background: var(--accent-orange);
            color: white;
        }

        .progress-status.completed {
            background: var(--accent-green);
            color: white;
        }

        .progress-status.error {
            background: var(--accent-red);
            color: white;
        }

        .progress-status.starting {
            background: var(--accent-cyan);
            color: white;
        }

        .progress-details {
            color: var(--text-secondary);
            font-size: 11px;
            line-height: 1.3;
        }

        .performance-metrics {
            background: rgba(88, 166, 255, 0.05);
            border: 1px solid rgba(88, 166, 255, 0.2);
            border-radius: 6px;
            padding: 8px;
            margin-top: 6px;
            font-size: 10px;
        }

        .performance-metrics .metric {
            display: flex;
            justify-content: space-between;
            margin-bottom: 2px;
        }

        .no-agent-selected {
            display: flex;
            align-items: center;
            justify-content: center;
            flex-direction: column;
            gap: 16px;
            height: 100%;
            color: var(--text-muted);
            text-align: center;
        }

        .no-agent-selected .icon {
            font-size: 48px;
            opacity: 0.5;
        }

        .typing-indicator {
            display: none;
            align-items: center;
            gap: 8px;
            padding: 12px 16px;
            background: var(--bg-tertiary);
            margin: 12px 20px;
            border-radius: 16px;
            font-size: 14px;
            color: var(--text-muted);
            flex-shrink: 0;
        }

        .typing-indicator.active { display: flex; }

        .typing-dots {
            display: flex;
            gap: 4px;
        }

        .typing-dot {
            width: 6px;
            height: 6px;
            border-radius: 50%;
            background: var(--text-muted);
            animation: typing 1.4s infinite;
        }

        .typing-dot:nth-child(2) { animation-delay: 0.2s; }
        .typing-dot:nth-child(3) { animation-delay: 0.4s; }

        @keyframes typing {
            0%, 60%, 100% { opacity: 0.3; }
            30% { opacity: 1; }
        }

        .system-info {
            margin-top: auto;
            padding: 12px;
            border-top: 1px solid var(--border-color);
            font-size: 11px;
            color: var(--text-muted);
            flex-shrink: 0;
        }

        .error-message {
            background: rgba(248, 81, 73, 0.1);
            border: 1px solid var(--accent-red);
            color: var(--accent-red);
            padding: 12px;
            border-radius: 6px;
            margin: 12px;
            font-size: 14px;
            position: fixed;
            bottom: 20px;
            right: 20px;
            z-index: 2000;
            max-width: 300px;
        }

        .event-detail-modal {
            position: fixed;
            top: 0;
            left: 0;
            right: 0;
            bottom: 0;
            background: rgba(0, 0, 0, 0.8);
            display: none;
            align-items: center;
            justify-content: center;
            z-index: 2000;
            padding: 20px;
        }

        .event-detail-modal.active {
            display: flex;
        }

        .event-detail-content {
            background: var(--bg-secondary);
            border: 1px solid var(--border-color);
            border-radius: 12px;
            max-width: 800px;
            max-height: 80vh;
            width: 100%;
            display: flex;
            flex-direction: column;
            overflow: hidden;
            box-shadow: var(--shadow);
        }

        .event-detail-header {
            padding: 20px 24px;
            border-bottom: 1px solid var(--border-color);
            background: var(--bg-tertiary);
            display: flex;
            align-items: center;
            justify-content: space-between;
            flex-shrink: 0;
        }

        .event-detail-title {
            font-size: 18px;
            font-weight: 600;
            color: var(--text-primary);
            display: flex;
            align-items: center;
            gap: 12px;
        }

        .event-detail-close {
            background: none;
            border: none;
            color: var(--text-muted);
            cursor: pointer;
            padding: 8px;
            border-radius: 6px;
            font-size: 20px;
            transition: all 0.2s;
        }

        .event-detail-close:hover {
            background: var(--bg-primary);
            color: var(--text-primary);
        }

        .event-detail-body {
            flex: 1;
            overflow-y: auto;
            padding: 24px;
            min-height: 0;
        }

        .event-section {
            margin-bottom: 24px;
        }

        .event-section-title {
            font-size: 14px;
            font-weight: 600;
            color: var(--accent-blue);
            text-transform: uppercase;
            letter-spacing: 0.5px;
            margin-bottom: 12px;
            padding-bottom: 6px;
            border-bottom: 1px solid var(--border-color);
        }

        .event-field {
            display: flex;
            justify-content: space-between;
            align-items: flex-start;
            padding: 8px 0;
            border-bottom: 1px solid rgba(48, 54, 61, 0.5);
            font-size: 14px;
        }

        .event-field:last-child {
            border-bottom: none;
        }

        .event-field-label {
            font-weight: 500;
            color: var(--text-secondary);
            min-width: 140px;
            flex-shrink: 0;
        }

        .event-field-value {
            color: var(--text-primary);
            flex: 1;
            text-align: right;
            word-break: break-word;
        }

        .event-field-value.json {
            background: var(--bg-primary);
            border-radius: 6px;
            padding: 8px;
            font-family: 'Consolas', 'Monaco', monospace;
            font-size: 12px;
            text-align: left;
            white-space: pre-wrap;
            max-height: 200px;
            overflow-y: auto;
        }

        .event-status-badge {
            padding: 4px 8px;
            border-radius: 4px;
            font-size: 12px;
            font-weight: 500;
        }

        .event-status-badge.completed {
            background: var(--accent-green);
            color: white;
        }

        .event-status-badge.running {
            background: var(--accent-orange);
            color: white;
        }

        .event-status-badge.failed {
            background: var(--accent-red);
            color: white;
        }

        .progress-item {
            cursor: pointer;
            transition: all 0.2s;
        }

        .progress-item:hover {
            transform: translateY(-2px);
            box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);
        }

        .thinking-step {
            cursor: pointer;
            transition: all 0.2s;
        }

        .thinking-step:hover {
            transform: translateY(-1px);
            box-shadow: 0 2px 8px rgba(0, 0, 0, 0.15);
        }
    </style>
</head>
<body>

<div class="api-key-modal" id="api-key-modal">
    <div class="api-key-content">
        <div class="api-key-title">🔐 Enter API Key</div>
        <div class="api-key-description">
            Please enter your API key to access the agent. You can find this key in your agent registration details.
        </div>
        <input type="text" class="api-key-input" id="api-key-input"
               placeholder="tbk_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx">
        <button class="api-key-button" id="api-key-submit">Connect</button>
    </div>
</div>

<div class="header">
    <div class="logo">
        <span>🤖</span>
        <span>Agent Registry</span>
    </div>
    <div class="header-controls">
        <button class="panel-toggle active" id="sidebar-toggle">📋 Agents</button>
        <button class="panel-toggle active" id="progress-toggle">📊 Progress</button>
        <div class="status-indicator disconnected" id="connection-status">
            <div class="status-dot"></div>
            <span>Connecting...</span>
        </div>
    </div>
</div>

<div class="mobile-tabs">
    <div class="mobile-tab active" data-tab="chat">💬 Chat</div>
    <div class="mobile-tab" data-tab="agents">📋 Agents</div>
    <div class="mobile-tab" data-tab="progress">📊 Progress</div>
</div>

<div class="main-container">
    <!-- Agents Sidebar -->
    <div class="sidebar" id="sidebar">
        <div class="sidebar-header">
            <div class="sidebar-title">Available Agents</div>
            <button class="collapse-btn" id="sidebar-collapse">◀</button>
        </div>
        <div class="agents-list">
            <div id="agents-container">
                <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                    Loading agents...
                </div>
            </div>
        </div>
        <div class="system-info">
            <div>Registry Server</div>
            <div id="server-info">ws://localhost:8080</div>
        </div>
    </div>

    <!-- Chat Area -->
    <div class="chat-area">
        <div class="chat-header">
            <div class="chat-title" id="chat-title">Select an Agent</div>
            <div class="chat-subtitle" id="chat-subtitle">Choose an agent from the sidebar to start chatting</div>
        </div>

        <div class="messages-container" id="messages-container">
            <div class="no-agent-selected">
                <div class="icon">💬</div>
                <div>Select an agent to start a conversation</div>
            </div>
        </div>

        <div class="typing-indicator" id="typing-indicator">
            <span>Agent is thinking</span>
            <div class="typing-dots">
                <div class="typing-dot"></div>
                <div class="typing-dot"></div>
                <div class="typing-dot"></div>
            </div>
        </div>

        <div class="message-input">
            <input type="text" class="input-field" id="message-input"
                   placeholder="Type your message..." disabled>
            <button class="send-button" id="send-button" disabled>Send</button>
        </div>
    </div>
    <!-- Progress Panel -->
    <div class="progress-panel" id="progress-panel">
        <div class="progress-header">
            <span>Live Progress</span>
            <button class="collapse-btn" id="progress-collapse">▶</button>
        </div>
        <div class="panel-content" id="progress-content">
            <div class="progress-section">
                <div class="section-title">Current Status</div>
                <div id="current-status">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">
                        No active execution
                    </div>
                </div>
            </div>

            <div class="progress-section">
                <div class="section-title">Performance Metrics</div>
                <div id="performance-metrics">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 10px;">
                        No metrics available
                    </div>
                </div>
            </div>

            <div class="progress-section">
                <div class="section-title">Meta Tools History</div>
                <div id="meta-tools-history">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 10px;">
                        No meta-tool activity
                    </div>
                </div>
            </div>

            <div class="progress-section">
                <div class="section-title">System Events</div>
                <div id="system-events">
                    <div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 10px;">
                        System idle
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>

<script unSave="true">



    class AgentRegistryUI {
        constructor() {
            this.ws = null;
            this.currentAgent = null;
            this.sessionId = 'ui_session_' + Math.random().toString(36).substr(2, 9);
            this.isConnected = false;
            this.reconnectAttempts = 0;
            this.apiKey = null;
            this.maxReconnectAttempts = 10;
            this.reconnectDelay = 1000;

            this.panelStates = {
                sidebar: true,
                progress: true,
                mobile: 'chat'
            };

            this.agents = new Map();
            this.currentExecution = null;

            // NEW: Enhanced progress tracking
            this.progressHistory = [];
            this.maxProgressHistory = 200;
            this.expandedProgressItem = null;
            this.currentPerformanceMetrics = null;
            this.currentOutline = null;

            this.elements = {
                connectionStatus: document.getElementById('connection-status'),
                agentsContainer: document.getElementById('agents-container'),
                chatTitle: document.getElementById('chat-title'),
                chatSubtitle: document.getElementById('chat-subtitle'),
                messagesContainer: document.getElementById('messages-container'),
                messageInput: document.getElementById('message-input'),
                sendButton: document.getElementById('send-button'),
                typingIndicator: document.getElementById('typing-indicator'),
                serverInfo: document.getElementById('server-info'),

                // API Key elements
                apiKeyModal: document.getElementById('api-key-modal'),
                apiKeyInput: document.getElementById('api-key-input'),
                apiKeySubmit: document.getElementById('api-key-submit'),

                // Panel control elements
                sidebarToggle: document.getElementById('sidebar-toggle'),
                progressToggle: document.getElementById('progress-toggle'),
                sidebarCollapse: document.getElementById('sidebar-collapse'),
                progressCollapse: document.getElementById('progress-collapse'),
                mainContainer: document.querySelector('.main-container'),
                sidebar: document.getElementById('sidebar'),
                progressPanel: document.getElementById('progress-panel'),
                progressContent: document.getElementById('progress-content')
            };

            // Enhanced cleanup timer
            setInterval(() => {
                if (this.isTyping && this.currentExecution) {
                    const timeSinceLastUpdate = Date.now() - this.currentExecution.lastUpdate;
                    if (timeSinceLastUpdate > 30000) {
                        console.log('🧹 Cleanup: Hiding stuck typing indicator');
                        this.showTypingIndicator(false);
                        this.currentExecution = null;
                        this.updateCurrentStatusToIdle();
                    }
                }
            }, 5000);

            this.init();
        }

        init() {
            this.setupEventListeners();
            this.setupPanelControls();
            this.initializeProgressPanel();
            this.showApiKeyModal();
        }

        // NEW: Initialize the refactored progress panel
        initializeProgressPanel() {
            if (this.elements.progressContent) {
                this.elements.progressContent.innerHTML = `
                <div class="progress-section metrics-section">
                    <div class="section-title expandable-section" onclick="window.agentUI.toggleSection('metrics')">
                        <span>📊 Performance Metrics</span>
                        <span class="section-toggle">▼</span>
                    </div>
                    <div class="section-content" id="performance-metrics">
                        <div class="no-data">No metrics available</div>
                    </div>
                </div>

                <div class="progress-section outline-section">
                    <div class="section-title expandable-section" onclick="window.agentUI.toggleSection('outline')">
                        <span>🗺️ Execution Outline & Context</span>
                        <span class="section-toggle">▼</span>
                    </div>
                    <div class="section-content" id="execution-outline">
                        <div class="no-data">No outline available</div>
                    </div>
                </div>

                <div class="progress-section status-history-section">
                    <div class="section-title expandable-section" onclick="window.agentUI.toggleSection('status')">
                        <span>⚡ Status & History</span>
                        <span class="section-toggle">▼</span>
                    </div>
                    <div class="section-content expanded" id="status-history">
                        <div class="no-data">No active execution</div>
                    </div>
                </div>
            `;
            }
        }

        // NEW: Toggle progress panel sections
        toggleSection(sectionName) {
            const section = document.querySelector(`.${sectionName}-section .section-content`);
            const toggle = document.querySelector(`.${sectionName}-section .section-toggle`);

            if (!section || !toggle) return;

            const isExpanded = section.classList.contains('expanded');

            if (isExpanded) {
                section.classList.remove('expanded');
                toggle.textContent = '▼';
            } else {
                section.classList.add('expanded');
                toggle.textContent = '▲';
            }
        }

        // REFACTORED: Main message handler with unified progress system
        handleWebSocketMessage(data) {
            console.log('WebSocket message received:', data);

            if (data.event === 'execution_progress') {
                const executionData = data.data;
                if (executionData && executionData.payload) {
                    this.handleUnifiedProgressEvent(executionData);
                }
                return;
            }

            if (data.request_id && data.payload) {
                this.handleUnifiedProgressEvent(data);
                return;
            }

            if (data.event) {
                this.handleRegistryEvent(data);
                return;
            }

            console.log('Unhandled message format:', data);
        }

        // NEW: Unified progress event handler
        // REPLACE the existing handleUnifiedProgressEvent method
        handleUnifiedProgressEvent(eventData) {
            const payload = eventData.payload;
            const eventType = payload.event_type;
            const isFinal = eventData.is_final;
            const requestId = eventData.request_id;

            console.log(`🎯 Processing Event: ${eventType}`, payload);

            // Handle final events
            if (isFinal || eventType === 'execution_complete' || payload.status === 'completed') {
                this.showTypingIndicator(false);

                const result = payload.metadata?.result || payload.result || payload.response || payload.output;
                if (result && typeof result === 'string' && result.trim()) {
                    this.addMessage('agent', result);
                }

                this.currentExecution = null;
                this.updateCurrentStatusToIdle();
                return;
            }

            // Initialize execution tracking
            if (!this.currentExecution) {
                this.currentExecution = {
                    requestId,
                    startTime: Date.now(),
                    events: [],
                    lastUpdate: Date.now()
                };
                this.showTypingIndicator(true);
            }

            // ADD: Store ALL events in progress history
            this.addToProgressHistory(payload);

            // Handle chat integration for important events
            this.handleChatIntegration(payload);

            // Update performance metrics
            this.updatePerformanceMetricsFromEvent(payload);

            // Update execution outline
            this.updateExecutionOutlineFromEvent(payload);

            // Refresh status history (shows all events)
            this.refreshStatusHistory();

            // Update current execution
            if (this.currentExecution) {
                this.currentExecution.events.push({...payload, timestamp: Date.now()});
                this.currentExecution.lastUpdate = Date.now();
            }
        }

        // NEW: Add event to progress history
        // UPDATE the addToProgressHistory method to ensure all events are captured
        addToProgressHistory(payload) {
        const irrelevantEventTypes = ['node_phase', 'node_enter']; // Fügen Sie hier weitere Typen hinzu, falls nötig

    // Prüfen, ob der Event-Typ in der Liste der irrelevanten Typen ist
    if (irrelevantEventTypes.includes(payload.event_type)) {
        // Optional: Hier könnte man das Event kurz an anderer Stelle anzeigen,
        // aber wir speichern es nicht im langfristigen Verlauf.
        console.log(`📝 Skipping storage for irrelevant event: ${payload.event_type}`);
        return; // Die Funktion hier beenden, um das Speichern zu verhindern
    }

            // Generate consistent ID for events
            const eventId = payload.event_id || `event_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;

            const historyItem = {
                ...payload,
                timestamp: payload.timestamp || Date.now(),
                id: eventId
            };

            // Remove any existing event with same ID to avoid duplicates
            this.progressHistory = this.progressHistory.filter(item => item.id !== eventId);

            this.progressHistory.unshift(historyItem);

            if (this.progressHistory.length - 10 > this.maxProgressHistory) {
                this.progressHistory = this.progressHistory.slice(0, this.maxProgressHistory-50);
            }

            console.log(`📝 Added to progress history: ${payload.event_type}`, historyItem);
        }

        // NEW: Refresh unified status history display
        refreshStatusHistory() {
            const container = document.getElementById('status-history');
            if (!container) return;

            if (this.progressHistory.length === 0) {
                container.innerHTML = '<div class="no-data">No events recorded</div>';
                return;
            }

            container.innerHTML = '';

            this.progressHistory.forEach((event, index) => {
                const eventElement = this.createExpandableProgressItem(event, index === 0);
                container.appendChild(eventElement);
            });
        }

        // NEW: Create expandable progress item (only one expandable at a time)
// UPDATE the createExpandableProgressItem method to show more LLM details
        createExpandableProgressItem(event, isLatest = false) {
            const div = document.createElement('div');
            div.className = `progress-item expandable ${isLatest ? 'latest' : ''} ${event.event_type}`;
            div.setAttribute('data-event-id', event.id);

            const icon = this.getEventIcon(event.event_type, event.status);
            const title = this.getDisplayAction(event.event_type, event);
            const timestamp = new Date((event.timestamp || Date.now()) * (event.timestamp > 10000000000 ? 1 : 1000)).toLocaleTimeString();
            const status = event.status || 'unknown';

            // ADD: Special summary for LLM calls
            let summaryDetails = '';
            if (event.node_name) summaryDetails += `${event.node_name} • `;
            summaryDetails += timestamp;

            if (event.event_type === 'llm_call') {
                if (event.llm_temperature !== undefined) summaryDetails += ` • Temp: ${event.llm_temperature}`;
                if (event.llm_total_tokens) summaryDetails += ` • ${event.llm_total_tokens} tokens`;
                if (event.llm_cost) summaryDetails += ` • $${event.llm_cost.toFixed(4)}`;
                if (event.duration) summaryDetails += ` • ${event.duration.toFixed(2)}s`;
            } else {
                if (event.duration) summaryDetails += ` • ${event.duration.toFixed(2)}s`;
            }

            div.innerHTML = `
        <div class="progress-item-header" onclick="window.agentUI.toggleProgressItem('${event.id}')">
            <div class="progress-icon">${icon}</div>
            <div class="progress-title">${title}</div>
            <div class="progress-meta">
                <span class="progress-status ${status}">${status}</span>
                <span class="expand-indicator">▼</span>
            </div>
        </div>
        <div class="progress-summary">
            ${summaryDetails}
        </div>
        <div class="progress-item-expanded" id="expanded-${event.id}">
            ${this.createExpandedEventContent(event)}
        </div>
    `;

            return div;
        }
        // NEW: Toggle progress item (only one at a time)
        toggleProgressItem(eventId) {
            if (this.expandedProgressItem && this.expandedProgressItem !== eventId) {
                this.closeProgressItem(this.expandedProgressItem);
            }

            const expandedContent = document.getElementById(`expanded-${eventId}`);
            const progressItem = document.querySelector(`[data-event-id="${eventId}"]`);
            const indicator = progressItem?.querySelector('.expand-indicator');

            if (!expandedContent || !progressItem) return;

            const isExpanded = expandedContent.classList.contains('active');

            if (isExpanded) {
                expandedContent.classList.remove('active');
                progressItem.classList.remove('expanded');
                if (indicator) indicator.textContent = '▼';
                this.expandedProgressItem = null;
            } else {
                expandedContent.classList.add('active');
                progressItem.classList.add('expanded');
                if (indicator) indicator.textContent = '▲';
                this.expandedProgressItem = eventId;
            }
        }

        // NEW: Close progress item
        closeProgressItem(eventId) {
            const expandedContent = document.getElementById(`expanded-${eventId}`);
            const progressItem = document.querySelector(`[data-event-id="${eventId}"]`);
            const indicator = progressItem?.querySelector('.expand-indicator');

            if (expandedContent) expandedContent.classList.remove('active');
            if (progressItem) progressItem.classList.remove('expanded');
            if (indicator) indicator.textContent = '▼';
        }

        // NEW: Create detailed expanded content
// ADD this method to create comprehensive event details
        createExpandedEventContent(event) {
            const sections = [];

            // Core Information
            const coreInfo = this.extractCoreFields(event);
            if (Object.keys(coreInfo).length > 0) {
                sections.push(this.createEventSection('Core Information', coreInfo));
            }

            // Timing Information
            const timingInfo = this.extractTimingFields(event);
            if (Object.keys(timingInfo).length > 0) {
                sections.push(this.createEventSection('Timing & Status', timingInfo));
            }

            // LLM Information
            const llmInfo = this.extractLLMFields(event);
            if (Object.keys(llmInfo).length > 0) {
                sections.push(this.createEventSection('LLM Details', llmInfo));
            }

            // Tool Information
            const toolInfo = this.extractToolFields(event);
            if (Object.keys(toolInfo).length > 0) {
                sections.push(this.createEventSection('Tool Details', toolInfo));
            }

            // Performance Information
            const perfInfo = this.extractPerformanceFields(event);
            if (Object.keys(perfInfo).length > 0) {
                sections.push(this.createEventSection('Performance', perfInfo));
            }

            // Reasoning Context
            const reasoningInfo = this.extractReasoningFields(event);
            if (Object.keys(reasoningInfo).length > 0) {
                sections.push(this.createEventSection('Reasoning Context', reasoningInfo));
            }

            // Error Information
            const errorInfo = this.extractErrorFields(event);
            if (Object.keys(errorInfo).length > 0) {
                sections.push(this.createEventSection('Error Details', errorInfo));
            }

            // Raw Data
            const rawData = this.extractRawDataFields(event);
            if (Object.keys(rawData).length > 0) {
                sections.push(this.createEventSection('Raw Data', rawData));
            }

            return sections.join('') || '<div class="no-expanded-data">No detailed information available</div>';
        }

        // NEW: Create event section for expanded view
        createEventSection(title, fields) {
            const fieldsHtml = Object.entries(fields)
                .map(([key, value]) => {
                    if (typeof value === 'object' && value.type === 'json') {
                        return `
                        <div class="event-field">
                            <div class="event-field-label">${key}:</div>
                            <div class="event-field-value json">${value.value}</div>
                        </div>
                    `;
                    } else {
                        return `
                        <div class="event-field">
                            <div class="event-field-label">${key}:</div>
                            <div class="event-field-value">${value}</div>
                        </div>
                    `;
                    }
                })
                .join('');

            return `
            <div class="expanded-section">
                <div class="expanded-section-title">${title}</div>
                ${fieldsHtml}
            </div>
        `;
        }

        // ENHANCED: Chat integration for reasoning loops and task execution
        handleChatIntegration(payload) {
            const eventType = payload.event_type;
            const metadata = payload.metadata || {};
            switch (eventType) {
                case 'reasoning_loop':
                    if (metadata.outline_step && metadata.outline_total) {
                        this.handleOutlineStepInChat(payload);
                    }
                    break;
                case 'outline_created':
                    this.handleOutlineCreatedInChat(payload);
                    break;
                case 'task_start':
                case 'task_complete':
                case 'task_error':
                    this.handleTaskProgressInChat(payload);
                    break;
                case 'plan_created':
                    this.handlePlanCreatedInChat(payload);
                    break;
                case 'tool_call':
                    // Only show important tool calls in chat
                    if (payload.tool_name && !payload.tool_name.includes('internal')) {
                        this.handleToolCallInChat(payload);
                    }
                    break;
            }
        }

        // ADD this method for plan creation
handlePlanCreatedInChat(payload) {
    const metadata = payload.metadata || {};
    const planName = metadata.plan_name || 'Execution Plan';
    const taskCount = metadata.task_count || 0;
    const strategy = metadata.strategy || 'sequential';

    const planDiv = document.createElement('div');
    planDiv.className = 'thinking-step plan-created';
    planDiv.innerHTML = `
        <div class="thinking-step-header">
            <span>📋</span>
            <span>${planName} Created</span>
            <span class="step-status completed">Ready</span>
        </div>
        <div class="thinking-step-content">
            <div class="plan-details">
                <div class="plan-info">
                    <span class="plan-item">Tasks: ${taskCount}</span>
                    <span class="plan-item">Strategy: ${strategy}</span>
                </div>
                <div class="plan-ready">
                    <em>🚀 Plan ready for execution</em>
                </div>
            </div>
        </div>
    `;

    this.elements.messagesContainer.appendChild(planDiv);
    this.scrollToBottom();
}

        // ADD this new method for outline step progress
handleOutlineStepInChat(payload) {
    const metadata = payload.metadata || {};
    const outlineStep = metadata.outline_step || 0;
    const outlineTotal = metadata.outline_total || 0;
    const loopNumber = metadata.loop_number || 0;
    const status = payload.status || 'running';

    if (outlineStep === 0 || outlineTotal === 0) return;

    const progressPercentage = Math.round((outlineStep / outlineTotal) * 100);
    const isCompleted = status === 'completed';

    const stepDiv = document.createElement('div');
    stepDiv.className = `thinking-step outline-step ${isCompleted ? 'completed' : 'running'}`;

    let stepTitle = `Outline Step ${outlineStep} of ${outlineTotal}`;
    let stepIcon = isCompleted ? '✅' : '🗺️';
    let stepStatus = isCompleted ? 'Completed' : 'In Progress';

    stepDiv.innerHTML = `
        <div class="thinking-step-header">
            <span>${stepIcon}</span>
            <span>${stepTitle}</span>
            <span class="step-status ${status}">${stepStatus}</span>
        </div>
        <div class="thinking-step-content">
            <div class="outline-progress">
                <div class="progress-info">
                    <span class="progress-text">Execution Progress</span>
                    <span class="progress-percentage">${progressPercentage}%</span>
                </div>
                <div class="progress-bar-container">
                    <div class="progress-bar">
                        <div class="progress-bar-fill" style="width: ${progressPercentage}%"></div>
                    </div>
                </div>
                ${isCompleted ?
                    '<div class="step-completed">This execution step is now complete</div>' :
                    '<div class="step-working">Working on this step...</div>'
                }
            </div>

            ${metadata.context_size || metadata.task_stack_size ? `
                <div class="context-info">
                    ${metadata.context_size ? `<span class="context-item">Context: ${metadata.context_size}</span>` : ''}
                    ${metadata.task_stack_size ? `<span class="context-item">Tasks: ${metadata.task_stack_size}</span>` : ''}
                </div>
            ` : ''}
        </div>
    `;

    this.elements.messagesContainer.appendChild(stepDiv);
    this.scrollToBottom();
}


        // NEW: Handle outline creation with detailed information
// REPLACE the existing handleOutlineCreatedInChat method
handleOutlineCreatedInChat(payload) {
    const metadata = payload.metadata || {};
    const outline = metadata.outline;

    if (!outline) return;

    const outlineDiv = document.createElement('div');
    outlineDiv.className = 'thinking-step outline-created';
    outlineDiv.innerHTML = `
        <div class="thinking-step-header">
            <span>📋</span>
            <span>Execution Plan Created</span>
            <span class="step-status completed">Ready</span>
        </div>
        <div class="thinking-step-content">
            <div class="outline-content">
                ${this.formatOutlineForChat(outline)}
            </div>
            <div class="outline-ready">
                <em>✨ Ready to execute plan step by step</em>
            </div>
        </div>
    `;

    this.elements.messagesContainer.appendChild(outlineDiv);
    this.scrollToBottom();
}

        // NEW: Handle task execution progress cleanly
        handleTaskProgressInChat(payload) {
            const eventType = payload.event_type;
            const taskId = payload.task_id;
            const metadata = payload.metadata || {};
            const description = metadata.description || 'Task execution';
            const taskType = metadata.type || 'Task';
            const priority = metadata.priority || 'normal';

            let icon = '📋';
            let status = '';
            let statusClass = 'running';

            if (eventType === 'task_start') {
                icon = '▶️';
                status = 'Starting';
                statusClass = 'starting';
            } else if (eventType === 'task_complete') {
                icon = '✅';
                status = 'Completed';
                statusClass = 'completed';
            } else if (eventType === 'task_error') {
                icon = '❌';
                status = 'Failed';
                statusClass = 'error';
            }

            const taskDiv = document.createElement('div');
            taskDiv.className = `thinking-step task-progress ${statusClass}`;
            taskDiv.innerHTML = `
            <div class="thinking-step-header">
                <span>${icon}</span>
                <span>${taskType} ${status}</span>
                <span class="priority-badge ${priority}">${priority}</span>
            </div>
            <div class="thinking-step-content">
                <div class="task-description">${description}</div>
                ${payload.duration ? `<div class="task-timing">Duration: ${payload.duration.toFixed(2)}s</div>` : ''}
                ${eventType === 'task_error' && payload.error_details?.message ?
                `<div class="task-error">Error: ${payload.error_details.message}</div>` : ''}
            </div>
        `;

            this.elements.messagesContainer.appendChild(taskDiv);
            this.scrollToBottom();
        }

        // NEW: Handle tool calls in chat
        handleToolCallInChat(payload) {
            const toolName = payload.tool_name;
            const status = payload.status;

            if (status === 'running') return; // Only show completed tool calls

            const toolDiv = document.createElement('div');
            toolDiv.className = `thinking-step tool-call ${status}`;
            toolDiv.innerHTML = `
            <div class="thinking-step-header">
                <span>🔧</span>
                <span>Used ${toolName}</span>
                <span class="tool-status ${status}">${status}</span>
            </div>
            <div class="thinking-step-content">
                <div class="tool-result">
                    ${status === 'completed' ? 'Tool executed successfully' : 'Tool execution failed'}
                    ${payload.duration ? ` in ${payload.duration.toFixed(2)}s` : ''}
                </div>
            </div>
        `;

            this.elements.messagesContainer.appendChild(toolDiv);
            this.scrollToBottom();
        }

        // NEW: Format outline for chat display
        formatOutlineForChat(outline) {
            if (typeof outline === 'string') {
                return `<div class="outline-text">${outline}</div>`;
            }

            if (Array.isArray(outline)) {
                return `
                <div class="outline-steps">
                    ${outline.map((step, index) =>
                    `<div class="outline-step">
                            <span class="step-number">${index + 1}.</span>
                            <span class="step-text">${step}</span>
                        </div>`
                ).join('')}
                </div>
            `;
            }

            return '<div class="outline-text">Execution plan created</div>';
        }

        // NEW: Create progress bar
        createProgressBar(current, total) {
            if (!total || total === 0) return '';

            const percentage = Math.round((current / total) * 100);

            return `
            <div class="progress-bar-container">
                <div class="progress-bar-info">
                    <span>Progress</span>
                    <span>${percentage}%</span>
                </div>
                <div class="progress-bar">
                    <div class="progress-bar-fill" style="width: ${percentage}%"></div>
                </div>
            </div>
        `;
        }

        // ENHANCED: Update performance metrics
        updatePerformanceMetricsFromEvent(payload) {
            const metadata = payload.metadata || {};
            const performance = metadata.performance_metrics;

            if (performance && Object.keys(performance).length > 0) {
                this.currentPerformanceMetrics = performance;
                this.refreshPerformanceMetrics();
            }
        }

        // NEW: Refresh performance metrics display
        refreshPerformanceMetrics() {
            const container = document.getElementById('performance-metrics');
            if (!container || !this.currentPerformanceMetrics) return;

            const metrics = {
                'Action Efficiency': `${Math.round((this.currentPerformanceMetrics.action_efficiency || 0) * 100)}%`,
                'Avg Loop Time': `${(this.currentPerformanceMetrics.avg_loop_time || 0).toFixed(1)}s`,
                'Progress Rate': `${Math.round((this.currentPerformanceMetrics.progress_rate || 0) * 100)}%`,
                'Total Loops': this.currentPerformanceMetrics.total_loops || 0,
                'Progress Loops': this.currentPerformanceMetrics.progress_loops || 0
            };

            container.innerHTML = `
            <div class="metrics-grid">
                ${Object.entries(metrics).map(([key, value]) => `
                    <div class="metric-card">
                        <div class="metric-label">${key}</div>
                        <div class="metric-value">${value}</div>
                    </div>
                `).join('')}
            </div>
        `;
        }

        // NEW: Update execution outline
        updateExecutionOutlineFromEvent(payload) {
            const eventType = payload.event_type;
            const metadata = payload.metadata || {};

            if (eventType === 'outline_created' || eventType === 'reasoning_loop') {
                const outlineContainer = document.getElementById('execution-outline');
                if (!outlineContainer) return;

                const outline = metadata.outline;
                const outlineStep = metadata.outline_step || 0;
                const outlineTotal = metadata.outline_total || 0;
                const contextSize = metadata.context_size || 0;
                const taskStackSize = metadata.task_stack_size || 0;

                outlineContainer.innerHTML = `
                <div class="outline-info">
                    <div class="context-metrics">
                        <div class="context-metric">
                            <span class="context-label">Context Size:</span>
                            <span class="context-value">${contextSize}</span>
                        </div>
                        <div class="context-metric">
                            <span class="context-label">Task Stack:</span>
                            <span class="context-value">${taskStackSize}</span>
                        </div>
                        <div class="context-metric">
                            <span class="context-label">Progress:</span>
                            <span class="context-value">${outlineStep}/${outlineTotal}</span>
                        </div>
                    </div>

                    ${outlineTotal > 0 ? this.createProgressBar(outlineStep, outlineTotal) : ''}
                </div>

                ${outline ? `
                    <div class="outline-details">
                        <div class="outline-title">Current Plan</div>
                        ${this.formatOutlineForChat(outline)}
                    </div>
                ` : ''}
            `;
            }
        }

        // Helper methods for field extraction (using existing implementations)
        extractCoreFields(event) {
            const fields = {};
            if (event.event_type) fields['Event Type'] = event.event_type.replace(/_/g, ' ').toUpperCase();
            if (event.node_name) fields['Node'] = event.node_name;
            if (event.agent_name) fields['Agent'] = event.agent_name;
            if (event.task_id) fields['Task ID'] = event.task_id;
            if (event.plan_id) fields['Plan ID'] = event.plan_id;
            if (event.timestamp) fields['Timestamp'] = new Date((event.timestamp > 10000000000 ? event.timestamp : event.timestamp * 1000)).toLocaleString();
            return fields;
        }

// REPLACE the existing extractLLMFields method
        extractLLMFields(event) {
            const fields = {};
            const metadata = event.metadata || {};

            if (event.llm_model) fields['Model'] = event.llm_model;
            if (event.llm_temperature !== undefined) fields['Temperature'] = event.llm_temperature;
            if (event.llm_prompt_tokens) fields['Prompt Tokens'] = event.llm_prompt_tokens.toLocaleString();
            if (event.llm_completion_tokens) fields['Completion Tokens'] = event.llm_completion_tokens.toLocaleString();
            if (event.llm_total_tokens) fields['Total Tokens'] = event.llm_total_tokens.toLocaleString();
            if (event.llm_cost) fields['Cost'] = `$${event.llm_cost.toFixed(4)}`;

            // ADD: Model preferences and metadata
            if (metadata.model_preference) fields['Model Preference'] = metadata.model_preference;

            return fields;
        }

        extractToolFields(event) {
            const fields = {};
            const metadata = event.metadata || {};

            if (event.tool_name) fields['Tool Name'] = event.tool_name;
            if (metadata.meta_tool_name) fields['Meta Tool Name'] = metadata.meta_tool_name;

            if (event.is_meta_tool !== null && event.is_meta_tool !== undefined) {
                fields['Is Meta Tool'] = event.is_meta_tool ? '✅ Yes' : '❌ No';
            }

            // ADD: Tool execution details
            if (metadata.execution_phase) fields['Execution Phase'] = metadata.execution_phase;
            if (metadata.reasoning_loop) fields['Reasoning Loop'] = metadata.reasoning_loop;
            if (metadata.parsed_args && metadata.parsed_args.confidence_level) {
                fields['Confidence Level'] = `${Math.round(metadata.parsed_args.confidence_level * 100)}%`;
            }

            return fields;
        }

// ADD these helper methods for comprehensive data extraction
        extractTimingFields(event) {
            const fields = {};

            if (event.status) {
                fields['Status'] = `<span class="event-status-badge ${event.status}">${event.status.toUpperCase()}</span>`;
            }
            if (event.success !== null && event.success !== undefined) {
                fields['Success'] = event.success ? '✅ Yes' : '❌ No';
            }
            if (event.timestamp) {
                fields['Timestamp'] = new Date((event.timestamp > 10000000000 ? event.timestamp : event.timestamp * 1000)).toLocaleString();
            }
            if (event.duration) {
                fields['Duration'] = `${event.duration.toFixed(3)}s`;
            }
            if (event.node_duration) {
                fields['Node Duration'] = `${event.node_duration.toFixed(3)}s`;
            }
            if (event.routing_decision) {
                fields['Next Step'] = event.routing_decision;
            }

            return fields;
        }

        extractErrorFields(event) {
            const fields = {};

            if (event.error_details) {
                const errorDetails = event.error_details;
                if (errorDetails.message) fields['Error Message'] = errorDetails.message;
                if (errorDetails.type) fields['Error Type'] = errorDetails.type;
                if (errorDetails.traceback) {
                    fields['Traceback'] = {
                        type: 'json',
                        value: errorDetails.traceback
                    };
                }
            }

            if (event.tool_error) {
                fields['Tool Error'] = event.tool_error;
            }

            return fields;
        }
// REPLACE the existing extractRawDataFields method
        extractRawDataFields(event) {
            const fields = {};

            // ADD: Full LLM Input/Output for LLM calls
            if (event.event_type === 'llm_call') {
                if (event.llm_input) {
                    fields['LLM Input (Full Prompt)'] = {
                        type: 'json',
                        value: event.llm_input
                    };
                }

                if (event.llm_output) {
                    fields['LLM Output (Response)'] = {
                        type: 'json',
                        value: event.llm_output
                    };
                }
            }

            // Show other raw data for tool calls
            if (event.tool_args && typeof event.tool_args === 'object') {
                fields['Tool Arguments'] = {
                    type: 'json',
                    value: JSON.stringify(event.tool_args, null, 2)
                };
            }

            if (event.tool_result) {
                const resultStr = typeof event.tool_result === 'string' ?
                    event.tool_result :
                    JSON.stringify(event.tool_result, null, 2);

                fields['Tool Result'] = {
                    type: 'json',
                    value: resultStr.length > 1000 ?
                        resultStr.substring(0, 1000) + '\\n\\n... [truncated]' :
                        resultStr
                };
            }

            return fields;
        }

        extractPerformanceFields(event) {
            const fields = {};
            const metadata = event.metadata || {};
            const performance = metadata.performance_metrics || {};

            if (performance.action_efficiency) fields['Action Efficiency'] = `${Math.round(performance.action_efficiency * 100)}%`;
            if (performance.avg_loop_time) fields['Avg Loop Time'] = `${performance.avg_loop_time.toFixed(2)}s`;
            if (performance.progress_rate) fields['Progress Rate'] = `${Math.round(performance.progress_rate * 100)}%`;

            return fields;
        }

        extractReasoningFields(event) {
            const fields = {};
            const metadata = event.metadata || {};

            if (metadata.outline_step && metadata.outline_total) {
                fields['Outline Progress'] = `${metadata.outline_step}/${metadata.outline_total}`;
            }
            if (metadata.loop_number) fields['Loop Number'] = metadata.loop_number;
            if (metadata.context_size) fields['Context Size'] = metadata.context_size.toLocaleString();
            if (metadata.task_stack_size) fields['Task Stack Size'] = metadata.task_stack_size;

            return fields;
        }

        extractMetadata(event) {
            const fields = {};
            const metadata = event.metadata || {};

            // Show complex data as JSON
            const complexFields = ['tool_args', 'tool_result', 'llm_input', 'llm_output', 'error_details'];

            for (const field of complexFields) {
                if (event[field] && typeof event[field] === 'object') {
                    fields[field.replace(/_/g, ' ').toUpperCase()] = {
                        type: 'json',
                        value: JSON.stringify(event[field], null, 2)
                    };
                }
            }

            return fields;
        }

        // Enhanced helper methods
        // REPLACE the existing getDisplayAction method
        getDisplayAction(eventType, payload) {
            const metadata = payload.metadata || {};
            switch (eventType) {
                case 'reasoning_loop':
                    const step = metadata.outline_step || 0;
                    const total = metadata.outline_total || 0;
                    return step > 0 ? `Reasoning Step ${step}/${total}` : 'Deep Reasoning';
                case 'task_start':
                    return `Starting: ${metadata.description || 'Task'}`;
                case 'task_complete':
                    return `Completed: ${metadata.description || 'Task'}`;
                case 'task_error':
                    return `Failed: ${metadata.description || 'Task'}`;
                case 'tool_call':
                    const status = payload.status || 'running';
                    const toolName = payload.tool_name || 'Unknown Tool';
                    return `${status === 'running' ? 'Calling' : 'Called'} ${toolName}`;

                case 'llm_call':
                    const llmStatus = payload.status || 'running';
                    const model = payload.llm_model || 'LLM';
                    const taskId = payload.task_id || '';

                    // Show more context for LLM calls
                    let displayText = `${llmStatus === 'running' ? '🔄 Calling' : '✅ Called'} ${model}`;
                    if (taskId && taskId !== 'unknown') {
                        displayText += ` (${taskId})`;
                    }
                    return displayText;
                case 'plan_created':
                    return `Plan: ${metadata.plan_name || 'Execution Plan'}`;
                case 'outline_created':
                    return 'Execution Outline Created';
                case 'node_enter':
                    return `Started: ${payload.node_name || 'Processing'}`;
                case 'node_exit':
                    return `Finished: ${payload.node_name || 'Processing'}`;
                case 'node_phase':
                    return `${payload.node_name || 'Node'}: ${payload.node_phase || 'Processing'}`;
                case 'execution_start':
                    return 'Execution Started';
                case 'execution_complete':
                    return 'Execution Complete';
                // ADD: Meta tool events
                case 'meta_tool_call':
                    const metaToolName = metadata.meta_tool_name || payload.tool_name || 'Meta Tool';
                    const metaStatus = payload.status || 'running';
                    return `${metaStatus === 'running' ? 'Using' : 'Used'} ${metaToolName.replace(/_/g, ' ')}`;
                // ADD: Error events
                case 'error':
                    return `Error in ${payload.node_name || 'System'}`;
                default:
                    return eventType.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase());
            }
        }

        // REPLACE the existing getEventIcon method
        getEventIcon(eventType, status) {
            if (status === 'error' || status === 'failed') return '❌';
            if (status === 'completed') return '✅';

            switch (eventType) {
                case 'reasoning_loop': return '🧠';
                case 'task_start': return '▶️';
                case 'task_complete': return '✅';
                case 'task_error': return '❌';
                case 'tool_call': return '🔧';
                case 'llm_call': return '💭';
                case 'plan_created': return '📋';
                case 'outline_created': return '🗺️';
                case 'node_enter': return '🚀';
                case 'node_exit': return '🏁';
                case 'node_phase': return '⚙️';
                case 'execution_start': return '🎬';
                case 'execution_complete': return '🎉';
                case 'meta_tool_call': return '🛠️';
                case 'error': return '🚨';
                default: return '⚡';
            }
        }

        updateCurrentStatusToIdle() {
            const container = document.getElementById('status-history');
            if (container && container.children.length === 0) {
                container.innerHTML = `
                <div class="progress-item idle-status">
                    <div class="progress-item-header">
                        <div class="progress-icon">💤</div>
                        <div class="progress-title">Ready & Waiting</div>
                        <div class="progress-meta">
                            <span class="progress-status idle">idle</span>
                        </div>
                    </div>
                    <div class="progress-summary">
                        Agent ready for next message • ${new Date().toLocaleTimeString()}
                    </div>
                </div>
            `;
            }
        }

        showTypingIndicator(show) {
            console.log(`💭 ${show ? 'Showing' : 'Hiding'} typing indicator`);
            this.elements.typingIndicator.classList.toggle('active', show);
            if (show) {
                this.elements.typingIndicator.scrollIntoView({ behavior: 'smooth', block: 'end' });
            }
            this.isTyping = show;
        }

        scrollToBottom() {
            if (this.elements.messagesContainer) {
                this.elements.messagesContainer.scrollTop = this.elements.messagesContainer.scrollHeight;
            }
        }

        showApiKeyModal() {
            const storedKey = localStorage.getItem('agent_registry_api_key');
            if (storedKey) {
                this.apiKey = storedKey;
                this.elements.apiKeyModal.style.display = 'none';
                this.connect();
            } else {
                this.elements.apiKeyModal.style.display = 'flex';
            }
        }

        async validateAndStoreApiKey() {
            const apiKey = this.elements.apiKeyInput.value.trim();
            if (!apiKey) {
                this.showError('Please enter an API key');
                return;
            }

            if (!apiKey.startsWith('tbk_')) {
                this.showError('Invalid API key format (should start with tbk_)');
                return;
            }

            this.apiKey = apiKey;
            this.elements.apiKeyModal.style.display = 'none';
            this.connect();
        }

        setupPanelControls() {
            this.elements.sidebarToggle?.addEventListener('click', () => this.togglePanel('sidebar'));
            this.elements.progressToggle?.addEventListener('click', () => this.togglePanel('progress'));
            this.elements.sidebarCollapse?.addEventListener('click', () => this.togglePanel('sidebar'));
            this.elements.progressCollapse?.addEventListener('click', () => this.togglePanel('progress'));

            const mobileTabs = document.querySelectorAll('.mobile-tab');
            if (mobileTabs.length > 0) {
                mobileTabs.forEach(tab => {
                    tab.addEventListener('click', () => this.switchMobileTab(tab.dataset.tab));
                });
            }

            this.setupResponsiveHandlers();
        }

        togglePanel(panel) {
            this.panelStates[panel] = !this.panelStates[panel];
            this.updatePanelStates();
        }

        updatePanelStates() {
            const { sidebar, progress } = this.panelStates;

            if (this.elements.mainContainer) {
                this.elements.mainContainer.classList.remove('sidebar-collapsed', 'progress-collapsed', 'both-collapsed');
                if (!sidebar && !progress) {
                    this.elements.mainContainer.classList.add('both-collapsed');
                } else if (!sidebar) {
                    this.elements.mainContainer.classList.add('sidebar-collapsed');
                } else if (!progress) {
                    this.elements.mainContainer.classList.add('progress-collapsed');
                }
            }

            if (this.elements.sidebar) this.elements.sidebar.classList.toggle('collapsed', !sidebar);
            if (this.elements.progressPanel) this.elements.progressPanel.classList.toggle('collapsed', !progress);

            if (this.elements.sidebarToggle) {
                this.elements.sidebarToggle.classList.toggle('active', sidebar);
                this.elements.sidebarToggle.textContent = sidebar ? '📋 Agents' : '📋';
            }

            if (this.elements.progressToggle) {
                this.elements.progressToggle.classList.toggle('active', progress);
                this.elements.progressToggle.textContent = progress ? '📊 Progress' : '📊';
            }

            if (this.elements.sidebarCollapse) this.elements.sidebarCollapse.textContent = sidebar ? '◀' : '▶';
            if (this.elements.progressCollapse) this.elements.progressCollapse.textContent = progress ? '▶' : '◀';

            if (this.elements.mainContainer) this.elements.mainContainer.offsetHeight;
        }

        handleWindowResize() {
            const chatArea = document.querySelector('.chat-area');
            const mainContainer = this.elements.mainContainer;

            if (chatArea && mainContainer) {
                const currentDisplay = mainContainer.style.display;
                mainContainer.style.display = 'none';
                mainContainer.offsetHeight;
                mainContainer.style.display = currentDisplay || '';
            }
        }

        switchMobileTab(tab) {
            this.panelStates.mobile = tab;

            const mobileTabs = document.querySelectorAll('.mobile-tab');
            if (mobileTabs.length > 0) {
                mobileTabs.forEach(t => t.classList.toggle('active', t.dataset.tab === tab));
            }

            const sidebarEl = document.querySelector('.sidebar');
            const chatAreaEl = document.querySelector('.chat-area');
            const progressPanelEl = document.querySelector('.progress-panel');

            if (sidebarEl) sidebarEl.style.display = tab === 'agents' ? 'flex' : 'none';
            if (chatAreaEl) chatAreaEl.style.display = tab === 'chat' ? 'flex' : 'none';
            if (progressPanelEl) progressPanelEl.style.display = tab === 'progress' ? 'flex' : 'none';
        }

        setupResponsiveHandlers() {
            const mediaQuery = window.matchMedia('(max-width: 768px)');
            const handleResponsive = (e) => {
                if (e.matches) {
                    this.switchMobileTab(this.panelStates.mobile);
                } else {
                    const panels = document.querySelectorAll('.sidebar, .chat-area, .progress-panel');
                    panels.forEach(panel => { if (panel) panel.style.display = ''; });
                }
            };

            if (mediaQuery.addEventListener) {
                mediaQuery.addEventListener('change', handleResponsive);
            } else {
                mediaQuery.addListener(handleResponsive);
            }
            handleResponsive(mediaQuery);
        }

        setupEventListeners() {
            this.elements.apiKeySubmit?.addEventListener('click', () => this.validateAndStoreApiKey());
            window.addEventListener('resize', () => this.handleWindowResize());
            this.elements.apiKeyInput?.addEventListener('keypress', (e) => {
                if (e.key === 'Enter') this.validateAndStoreApiKey();
            });
            this.elements.sendButton.addEventListener('click', () => this.sendMessage());
            this.elements.messageInput.addEventListener('keypress', (e) => {
                if (e.key === 'Enter' && !e.shiftKey && this.currentAgent) {
                    e.preventDefault();
                    this.sendMessage();
                }
            });

            document.addEventListener('visibilitychange', () => {
                if (!document.hidden && (!this.ws || this.ws.readyState === WebSocket.CLOSED)) {
                    this.connect();
                }
            });
        }

        connect() {
            if (this.ws && this.ws.readyState === WebSocket.OPEN) return;

            this.updateConnectionStatus('connecting', 'Connecting...');

            try {
                const isLocal = window.location.hostname === 'localhost' || window.location.hostname === '127.0.0.1';
                const wsProtocol = isLocal ? 'ws' : 'wss';
                const wsUrl = `${wsProtocol}://${window.location.host}/ws/registry/ui_connect`;
                this.ws = new WebSocket(wsUrl);

                // Heartbeat mechanism
                this.heartbeatInterval = setInterval(() => {
                    if (this.ws && this.ws.readyState === WebSocket.OPEN) {
                        this.ws.send(JSON.stringify({event: 'ping', data: {}}));
                    }
                }, 25000); // Send ping every 25 seconds

                this.ws.onopen = () => {
                    this.isConnected = true;
                    this.reconnectAttempts = 0;
                    this.updateConnectionStatus('connected', 'Connected');
                    console.log('Connected to Registry Server');
                };

                this.ws.onmessage = (event) => {
                    try {
                        const data = JSON.parse(event.data);
                        if (data.event === 'pong') {
                            // Handle pong response
                            return;
                        }
                        this.handleWebSocketMessage(data);
                    } catch (error) {
                        console.error('Message parse error:', error);
                    }
                };

                this.ws.onclose = (event) => {
                    this.isConnected = false;
                    if (this.heartbeatInterval) {
                        clearInterval(this.heartbeatInterval);
                    }
                    this.updateConnectionStatus('disconnected', 'Disconnected');
                    this.scheduleReconnection();
                };

                this.ws.onerror = (error) => {
                    console.error('WebSocket error:', error);
                    this.updateConnectionStatus('error', 'Connection Error');
                };

            } catch (error) {
                console.error('Connection error:', error);
                this.updateConnectionStatus('error', 'Connection Failed');
                this.scheduleReconnection();
            }
        }

        scheduleReconnection() {
            if (this.reconnectAttempts >= this.maxReconnectAttempts) {
                this.updateConnectionStatus('error', 'Connection Failed (Max attempts reached)');
                return;
            }

            this.reconnectAttempts++;
            const delay = Math.min(this.reconnectDelay * this.reconnectAttempts, 30000);

            this.updateConnectionStatus('connecting', `Reconnecting in ${delay/1000}s (attempt ${this.reconnectAttempts})`);

            setTimeout(() => {
                if (!this.isConnected) this.connect();
            }, delay);
        }

        updateConnectionStatus(status, text) {
            this.elements.connectionStatus.className = `status-indicator ${status}`;
            this.elements.connectionStatus.querySelector('span').textContent = text;
        }

        handleRegistryEvent(data) {
            const event = data.event;
            const payload = data.data || data;

            console.log(`📋 Registry Event: ${event}`, payload);

            switch (event) {
                case 'api_key_validation':
                    if (payload.valid) {
                        console.log('✅ API key validated successfully');
                    } else {
                        this.showError('❌ Invalid API key for this agent');
                        this.currentAgent = null;
                        this.elements.messageInput.disabled = true;
                        this.elements.sendButton.disabled = true;
                    }
                    break;
                case 'agents_list':
                    console.log('📝 Updating agents list:', payload.agents);
                    this.updateAgentsList(payload.agents);
                    break;
                case 'agent_registered':
                    console.log('🆕 Agent registered:', payload);
                    this.addAgent(payload);
                    break;
                case 'error':
                    console.error('❌ WebSocket error:', payload);
                    this.showError(payload.error || payload.message || 'Unknown error');
                    break;
                default:
                    console.log('❓ Unhandled registry event:', event, payload);
            }
        }

        updateAgentsList(agents) {
            this.elements.agentsContainer.innerHTML = '';

            if (!agents || agents.length === 0) {
                this.elements.agentsContainer.innerHTML = '<div style="color: var(--text-muted); font-size: 12px; text-align: center; padding: 20px;">No agents available</div>';
                return;
            }

            agents.forEach(agent => {
                this.agents.set(agent.public_agent_id, agent);
                const agentEl = this.createAgentElement(agent);
                this.elements.agentsContainer.appendChild(agentEl);
            });
        }

        createAgentElement(agent) {
            const div = document.createElement('div');
            div.className = 'agent-item';
            div.dataset.agentId = agent.public_agent_id;

            div.innerHTML = `
            <div class="agent-name">${agent.public_name}</div>
            <div class="agent-description">${agent.description || 'No description'}</div>
            <div class="agent-status ${agent.status}">
                <div class="status-dot"></div>
                <span>${agent.status.toUpperCase()}</span>
            </div>
        `;

            div.addEventListener('click', () => this.selectAgent(agent));
            return div;
        }

        selectAgent(agent) {
            if (!this.apiKey) {
                this.showError('Please set your API key first');
                return;
            }

            this.sendWebSocketMessage({
                event: 'validate_api_key',
                data: { public_agent_id: agent.public_agent_id, api_key: this.apiKey }
            });

            document.querySelectorAll('.agent-item').forEach(el => el.classList.remove('active'));
            document.querySelector(`[data-agent-id="${agent.public_agent_id}"]`)?.classList.add('active');

            this.currentAgent = agent;
            this.elements.chatTitle.textContent = agent.public_name;
            this.elements.chatSubtitle.textContent = agent.description || 'Ready for conversation';

            this.elements.messageInput.disabled = false;
            this.elements.sendButton.disabled = false;

            this.elements.messagesContainer.innerHTML = '';
            this.addMessage('agent', `Hello! I'm ${agent.public_name}. How can I help you?`);

            this.sendWebSocketMessage({
                event: 'subscribe_agent',
                data: { public_agent_id: agent.public_agent_id }
            });

            this.sendWebSocketMessage({
                event: 'get_agent_status',
                data: { public_agent_id: agent.public_agent_id }
            });

            // Reset progress panels
            this.progressHistory = [];
            this.refreshStatusHistory();
            const metricsContainer = document.getElementById('performance-metrics');
            if (metricsContainer) metricsContainer.innerHTML = '<div class="no-data">No metrics available</div>';
            const outlineContainer = document.getElementById('execution-outline');
            if (outlineContainer) outlineContainer.innerHTML = '<div class="no-data">No outline available</div>';
        }

        sendMessage() {
            if (!this.currentAgent || !this.elements.messageInput.value.trim()) return;

            const message = this.elements.messageInput.value.trim();
            this.addMessage('user', message);

            this.sendWebSocketMessage({
                event: 'chat_message',
                data: {
                    public_agent_id: this.currentAgent.public_agent_id,
                    message: message,
                    session_id: this.sessionId,
                    api_key: this.apiKey
                }
            });

            this.elements.messageInput.value = '';

            // Reset progress state
            this.progressHistory = [];
            this.expandedProgressItem = null;
            this.refreshStatusHistory();

            // Failsafe timeout
            setTimeout(() => {
                if (this.currentExecution) {
                    console.log('⏰ Timeout: Hiding typing indicator and resetting execution state');
                    this.showTypingIndicator(false);
                    this.currentExecution = null;
                    this.updateCurrentStatusToIdle();
                    this.showError('Agent response timeout - please try again');
                }
            }, 60000);
        }

        addMessage(sender, content) {
            const messageDiv = document.createElement('div');
            messageDiv.classList.add('message', sender);

            const avatar = document.createElement('div');
            avatar.classList.add('message-avatar');
            avatar.textContent = sender === 'user' ? 'U' : 'AI';

            const contentDiv = document.createElement('div');
            contentDiv.classList.add('message-content');

            if (sender === 'agent' && window.marked) {
                try {
                    contentDiv.innerHTML = marked.parse(content);
                } catch (error) {
                    contentDiv.textContent = content;
                }
            } else {
                contentDiv.textContent = content;
            }

            messageDiv.appendChild(avatar);
            messageDiv.appendChild(contentDiv);

            this.elements.messagesContainer.appendChild(messageDiv);
            this.elements.messagesContainer.scrollTop = this.elements.messagesContainer.scrollHeight;

            if (sender === 'agent') {
                this.showTypingIndicator(false);
                setTimeout(() => {
                    if (this.currentExecution) {
                        this.currentExecution = null;
                        this.updateCurrentStatusToIdle();
                    }
                }, 1000);
            }
        }

        showError(message) {
            const errorDiv = document.createElement('div');
            errorDiv.className = 'error-message';
            errorDiv.textContent = message;

            document.body.appendChild(errorDiv);
            setTimeout(() => {
                if (errorDiv.parentNode) {
                    errorDiv.parentNode.removeChild(errorDiv);
                }
            }, 5000);
        }

        sendWebSocketMessage(data) {
            if (this.ws && this.ws.readyState === WebSocket.OPEN) {
                this.ws.send(JSON.stringify(data));
            } else {
                console.warn('WebSocket not connected, cannot send message');
            }
        }

    }

    // Initialize UI when DOM is ready
    if (!window.TB) {
        document.addEventListener('DOMContentLoaded', () => {
            window.agentUI = new AgentRegistryUI();
        });
    } else {
        TB.once(() => {
            window.agentUI = new AgentRegistryUI();
        });
    }
</script>
</body>
</html>"""

registry

client

RegistryClient

Manages the client-side connection to the Registry Server with robust reconnection and long-running support.

Source code in toolboxv2/mods/registry/client.py
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
class RegistryClient:
    """Manages the client-side connection to the Registry Server with robust reconnection and long-running support."""

    def __init__(self, app: App):
        self.app = app

        # WebSocket connection
        self.ws: ws_client.WebSocketClientProtocol | None = None
        self.server_url: str | None = None

        # Task management
        self.connection_task: asyncio.Task | None = None
        self.ping_task: asyncio.Task | None = None
        self.message_handler_tasks: set[asyncio.Task] = set()
        self.progress_processor_task: asyncio.Task | None = None

        # Connection state
        self.is_connected = False
        self.should_reconnect = True
        self.reconnect_in_progress = False
        self.reconnect_attempts = 0
        self.max_reconnect_attempts = 10

        # Agent management
        self.local_agents: dict[str, Any] = {}
        self.registered_info: dict[str, AgentRegistered] = {}
        self.running_executions: dict[str, asyncio.Task] = {}
        self.persistent_callbacks: dict[str, Callable] = {}

        # Progress streaming (NO BATCHING - immediate streaming)
        self.progress_queues: dict[str, asyncio.Queue] = {}
        self.active_streams: set[str] = set()

        # Event handling
        self.custom_event_handlers: dict[str, Callable[[dict], Awaitable[None]]] = {}
        self.pending_registrations: dict[str, asyncio.Future] = {}
        self.registration_counter = 0

    # Utility Methods
    async def get_connection_status(self) -> dict[str, Any]:
        """Get detailed connection status information."""
        try:
            connection_status = {
                "is_connected": self.is_connected,
                "server_url": self.server_url,
                "reconnect_attempts": self.reconnect_attempts,
                "max_reconnect_attempts": self.max_reconnect_attempts,
                "should_reconnect": self.should_reconnect,
                "reconnect_in_progress": self.reconnect_in_progress,
                "websocket_state": None,
                "websocket_open": False,
                "tasks": {
                    "connection_task_running": self.connection_task and not self.connection_task.done(),
                    "ping_task_running": self.ping_task and not self.ping_task.done(),
                },
                "registered_agents_count": len(self.local_agents),
                "running_executions_count": len(self.running_executions),
                "pending_registrations_count": len(self.pending_registrations),
                "persistent_callbacks_count": len(self.persistent_callbacks),
                "last_ping_time": getattr(self, 'last_ping_time', None),
                "connection_uptime": None,
                "connection_established_at": getattr(self, 'connection_established_at', None),
            }

            # WebSocket specific status
            if self.ws:
                connection_status.update({
                    "websocket_state": str(self.ws.state.name) if hasattr(self.ws.state, 'name') else str(
                        self.ws.state),
                    "websocket_open": self.ws.open,
                    "websocket_closed": self.ws.closed,
                })

            # Calculate uptime
            if hasattr(self, 'connection_established_at') and self.connection_established_at:
                connection_status[
                    "connection_uptime"] = asyncio.get_event_loop().time() - self.connection_established_at

            return connection_status

        except Exception as e:
            self.app.print(f"Error getting connection status: {e}")
            return {
                "error": str(e),
                "is_connected": False,
                "server_url": self.server_url,
            }

    async def get_registered_agents(self) -> dict[str, AgentRegistered]:
        """Get all registered agents information."""
        try:
            agents_info = {}

            for agent_id, reg_info in self.registered_info.items():
                # Get agent instance if available
                agent_instance = self.local_agents.get(agent_id)

                # Create enhanced agent info
                agent_data = {
                    "registration_info": reg_info,
                    "agent_available": agent_instance is not None,
                    "agent_type": type(agent_instance).__name__ if agent_instance else "Unknown",
                    "has_progress_callback": hasattr(agent_instance, 'progress_callback') if agent_instance else False,
                    "supports_progress_callback": hasattr(agent_instance,
                                                          'set_progress_callback') if agent_instance else False,
                    "is_persistent_callback_active": agent_id in self.persistent_callbacks,
                    "registration_timestamp": getattr(reg_info, 'registration_timestamp', None),
                }

                # Add agent capabilities if available
                if agent_instance and hasattr(agent_instance, 'get_capabilities'):
                    try:
                        agent_data["capabilities"] = await agent_instance.get_capabilities()
                    except Exception as e:
                        agent_data["capabilities_error"] = str(e)

                agents_info[agent_id] = agent_data

            return agents_info

        except Exception as e:
            self.app.print(f"Error getting registered agents: {e}")
            return {}

    async def get_running_executions(self) -> dict[str, dict[str, Any]]:
        """Get information about currently running executions."""
        try:
            executions_info = {}

            for request_id, execution_task in self.running_executions.items():
                execution_info = {
                    "request_id": request_id,
                    "task_done": execution_task.done(),
                    "task_cancelled": execution_task.cancelled(),
                    "start_time": getattr(execution_task, 'start_time', None),
                    "running_time": None,
                    "task_exception": None,
                    "task_result": None,
                }

                # Calculate running time
                if hasattr(execution_task, 'start_time') and execution_task.start_time:
                    execution_info["running_time"] = asyncio.get_event_loop().time() - execution_task.start_time

                # Get task status details
                if execution_task.done():
                    try:
                        if execution_task.exception():
                            execution_info["task_exception"] = str(execution_task.exception())
                        else:
                            execution_info["task_result"] = "completed_successfully"
                    except Exception as e:
                        execution_info["task_status_error"] = str(e)

                executions_info[request_id] = execution_info

            return executions_info

        except Exception as e:
            self.app.print(f"Error getting running executions: {e}")
            return {}

    async def cancel_execution(self, request_id: str) -> bool:
        """Cancel a running execution."""
        try:
            if request_id not in self.running_executions:
                self.app.print(f"❌ Execution {request_id} not found")
                return False

            execution_task = self.running_executions[request_id]

            if execution_task.done():
                self.app.print(f"⚠️  Execution {request_id} already completed")
                return True

            # Cancel the task
            execution_task.cancel()

            try:
                # Wait a moment for graceful cancellation
                await asyncio.wait_for(execution_task, timeout=5.0)
            except asyncio.CancelledError:
                self.app.print(f"✅ Execution {request_id} cancelled successfully")
            except asyncio.TimeoutError:
                self.app.print(f"⚠️  Execution {request_id} cancellation timeout - may still be running")
            except Exception as e:
                self.app.print(f"⚠️  Execution {request_id} cancellation resulted in exception: {e}")

            # Send cancellation notice to server
            try:
                if self.is_connected and self.ws and self.ws.open:
                    cancellation_event = ProgressEvent(
                        event_type="execution_cancelled",
                        node_name="RegistryClient",
                        success=False,
                        metadata={
                            "request_id": request_id,
                            "cancellation_reason": "client_requested",
                            "timestamp": asyncio.get_event_loop().time()
                        }
                    )

                    cancellation_message = ExecutionResult(
                        request_id=request_id,
                        payload=cancellation_event.to_dict(),
                        is_final=True
                    )

                    await self._send_message('execution_result', cancellation_message.model_dump())

            except Exception as e:
                self.app.print(f"Failed to send cancellation notice to server: {e}")

            # Cleanup
            self.running_executions.pop(request_id, None)

            return True

        except Exception as e:
            self.app.print(f"Error cancelling execution {request_id}: {e}")
            return False

    async def health_check(self) -> bool:
        """Perform a health check of the connection."""
        try:
            # Basic connection checks
            if not self.is_connected:
                self.app.print("🔍 Health check: Not connected")
                return False

            if not self.ws or not self.ws.open:
                self.app.print("🔍 Health check: WebSocket not open")
                return False

            # Ping test
            try:
                pong_waiter = await self.ws.ping()
                await asyncio.wait_for(pong_waiter, timeout=10.0)

                # Update last ping time
                self.last_ping_time = asyncio.get_event_loop().time()

                # Test message sending
                test_message = WsMessage(
                    event='health_check',
                    data={
                        "timestamp": self.last_ping_time,
                        "client_id": getattr(self, 'client_id', 'unknown'),
                        "registered_agents": list(self.local_agents.keys()),
                        "running_executions": list(self.running_executions.keys())
                    }
                )

                await self.ws.send(test_message.model_dump_json())

                self.app.print("✅ Health check: Connection healthy")
                return True

            except asyncio.TimeoutError:
                self.app.print("❌ Health check: Ping timeout")
                return False
            except Exception as ping_error:
                self.app.print(f"❌ Health check: Ping failed - {ping_error}")
                return False

        except Exception as e:
            self.app.print(f"❌ Health check: Error - {e}")
            return False

    async def get_diagnostics(self) -> dict[str, Any]:
        """Get comprehensive diagnostic information."""
        try:
            diagnostics = {
                "connection_status": await self.get_connection_status(),
                "registered_agents": await self.get_registered_agents(),
                "running_executions": await self.get_running_executions(),
                "health_status": await self.health_check(),
                "system_info": {
                    "python_version": sys.version,
                    "asyncio_running": True,
                    "event_loop": str(asyncio.get_running_loop()),
                    "thread_name": threading.current_thread().name,
                },
                "performance_metrics": {
                    "total_messages_sent": getattr(self, 'total_messages_sent', 0),
                    "total_messages_received": getattr(self, 'total_messages_received', 0),
                    "total_reconnections": self.reconnect_attempts,
                    "total_registrations": len(self.registered_info),
                    "memory_usage": self._get_memory_usage(),
                },
                "error_log": getattr(self, 'recent_errors', []),
            }

            return diagnostics

        except Exception as e:
            return {
                "diagnostics_error": str(e),
                "timestamp": asyncio.get_event_loop().time()
            }

    def _get_memory_usage(self) -> dict[str, Any]:
        """Get memory usage information."""
        try:
            import psutil
            import os

            process = psutil.Process(os.getpid())
            memory_info = process.memory_info()

            return {
                "rss": memory_info.rss,
                "vms": memory_info.vms,
                "percent": process.memory_percent(),
                "available": psutil.virtual_memory().available,
            }
        except ImportError:
            return {"error": "psutil not available"}
        except Exception as e:
            return {"error": str(e)}

    async def cleanup_completed_executions(self):
        """Clean up completed execution tasks."""
        try:
            completed_tasks = []

            for request_id, task in self.running_executions.items():
                if task.done():
                    completed_tasks.append(request_id)

            for request_id in completed_tasks:
                self.running_executions.pop(request_id, None)
                self.app.print(f"🧹 Cleaned up completed execution: {request_id}")

            return len(completed_tasks)

        except Exception as e:
            self.app.print(f"Error during cleanup: {e}")
            return 0

    async def connect(self, server_url: str, timeout: float = 30.0):
        """Connect and start all background tasks."""
        if not ws_client:
            self.app.print("Websockets library not installed. Please run 'pip install websockets'")
            return False

        if self.ws and self.ws.open:
            self.app.print("Already connected to the registry server.")
            return True

        self.server_url = server_url
        self.should_reconnect = True
        self.reconnect_in_progress = False

        try:
            self.app.print(f"Connecting to Registry Server at {server_url}...")
            self.ws = await asyncio.wait_for(
                ws_client.connect(server_url),
                timeout=timeout
            )

            self.is_connected = True
            self.reconnect_attempts = 0

            # Start all background tasks
            await self._start_all_background_tasks()

            self.app.print(f"✅ Successfully connected and started all tasks")
            return True

        except asyncio.TimeoutError:
            self.app.print(f"❌ Connection timeout after {timeout}s")
            return False
        except Exception as e:
            self.app.print(f"❌ Connection failed: {e}")
            return False

    async def _start_all_background_tasks(self):
        """Start all background tasks needed for operation."""
        # Start connection listener
        self.connection_task = asyncio.create_task(self._listen())

        # Start ping task
        self.ping_task = asyncio.create_task(self._ping_loop())

        self.app.print("🚀 All background tasks started")
    async def _start_ping_task(self):
        """Start the ping/heartbeat task in the background."""
        if self.ping_task and not self.ping_task.done():
            return  # Already running

        self.ping_task = asyncio.create_task(self._ping_loop())

    async def _ping_loop(self):
        """Dedicated ping task that never blocks and has highest priority."""
        ping_interval = 20  # Less aggressive than server's 5s interval
        consecutive_failures = 0
        max_failures = 2

        while self.is_connected and self.should_reconnect:
            try:
                await asyncio.sleep(ping_interval)

                # Double-check connection state
                if not self.ws or not self.ws.open or self.ws.closed:
                    self.app.print("Ping task detected closed connection")
                    break

                try:
                    # Send ping with short timeout
                    pong_waiter = await self.ws.ping()
                    await asyncio.wait_for(pong_waiter, timeout=8.0)  # Less than server's 10s timeout

                    consecutive_failures = 0
                    self.app.print("📡 Heartbeat successful")

                except asyncio.TimeoutError:
                    consecutive_failures += 1
                    self.app.print(f"⚠️ Ping timeout ({consecutive_failures}/{max_failures})")

                    if consecutive_failures >= max_failures:
                        self.app.print("❌ Multiple ping timeouts - connection dead")
                        break

                except Exception as ping_error:
                    consecutive_failures += 1
                    self.app.print(f"❌ Ping error ({consecutive_failures}/{max_failures}): {ping_error}")

                    if consecutive_failures >= max_failures:
                        break

            except Exception as e:
                self.app.print(f"Ping loop error: {e}")
                break

        self.app.print("Ping task stopped")
        # Trigger reconnect if we should still be connected
        if self.should_reconnect and self.is_connected:
            asyncio.create_task(self._trigger_reconnect())

    async def _trigger_reconnect(self):
        """Trigger a reconnection attempt."""
        if self.reconnect_in_progress:
            return

        self.reconnect_in_progress = True
        self.is_connected = False

        try:
            if self.ws:
                with contextlib.suppress(Exception):
                    await self.ws.close()
                self.ws = None

            # Stop current tasks
            if self.connection_task and not self.connection_task.done():
                self.connection_task.cancel()
            if self.ping_task and not self.ping_task.done():
                self.ping_task.cancel()

            self.app.print("🔄 Attempting to reconnect...")
            await self._reconnect_with_backoff()

        finally:
            self.reconnect_in_progress = False

    async def _reconnect_with_backoff(self):
        """Reconnect with exponential backoff."""
        max_attempts = 10
        base_delay = 2
        max_delay = 300  # 5 minutes max

        for attempt in range(max_attempts):
            if not self.should_reconnect:
                break

            delay = min(base_delay * (2 ** attempt), max_delay)
            self.app.print(f"🔄 Reconnect attempt {attempt + 1}/{max_attempts} in {delay}s...")

            await asyncio.sleep(delay)

            try:
                if self.server_url:
                    self.ws = await ws_client.connect(self.server_url)
                    self.is_connected = True
                    self.reconnect_attempts = 0

                    # Restart tasks
                    self.connection_task = asyncio.create_task(self._listen())
                    await self._start_ping_task()

                    # Re-register agents
                    await self._reregister_agents()

                    self.app.print("✅ Reconnected successfully!")
                    return

            except Exception as e:
                self.app.print(f"❌ Reconnect attempt {attempt + 1} failed: {e}")

        self.app.print("❌ All reconnection attempts failed")
        self.should_reconnect = False

    async def _reregister_agents(self):
        """Re-register all local agents after reconnection."""
        if not self.registered_info:
            self.app.print("No agents to re-register")
            return

        self.app.print(f"Re-registering {len(self.registered_info)} agents...")

        for agent_id, reg_info in list(self.registered_info.items()):
            try:
                agent_instance = self.local_agents.get(agent_id)
                if not agent_instance:
                    continue

                # Create new registration (server will assign new IDs)
                new_reg_info = await self.register(
                    agent_instance,
                    reg_info.public_name,
                    self.local_agents.get(f"{agent_id}_description", "Re-registered agent")
                )

                if new_reg_info:
                    # Update stored information
                    old_agent_id = agent_id
                    new_agent_id = new_reg_info.public_agent_id

                    # Move agent to new ID
                    self.local_agents[new_agent_id] = self.local_agents.pop(old_agent_id)
                    self.registered_info[new_agent_id] = self.registered_info.pop(old_agent_id)

                    self.app.print(f"✅ Re-registered agent: {reg_info.public_name} (new ID: {new_agent_id})")
                else:
                    self.app.print(f"❌ Failed to re-register agent: {reg_info.public_name}")

            except Exception as e:
                self.app.print(f"Error re-registering agent {reg_info.public_name}: {e}")

        self.app.print("Agent re-registration completed")

    async def _create_persistent_progress_callback(self, request_id: str, agent_id: str):
        """Create progress callback with offline queuing capability."""
        progress_queue = asyncio.Queue(maxsize=100)  # Buffer for offline messages

        async def persistent_progress_callback(event: ProgressEvent):
            try:
                # Add to queue first
                try:
                    progress_queue.put_nowait((event, asyncio.get_event_loop().time()))
                except asyncio.QueueFull:
                    # Remove oldest item and add new one
                    try:
                        progress_queue.get_nowait()
                        progress_queue.put_nowait((event, asyncio.get_event_loop().time()))
                    except asyncio.QueueEmpty:
                        pass

                # Try to send immediately if connected
                if await self._check_connection_health():
                    try:
                        result = ExecutionResult(
                            request_id=request_id,
                            payload=event.to_dict(),
                            is_final=False
                        )
                        success = await self._send_message('execution_result', result.model_dump())
                        if success:
                            # Remove from queue since it was sent successfully
                            try:
                                progress_queue.get_nowait()
                            except asyncio.QueueEmpty:
                                pass
                            return
                    except Exception as e:
                        self.app.print(f"Progress send failed, queued: {e}")

                # If we get here, message is queued for later sending

            except Exception as e:
                self.app.print(f"Progress callback error: {e}")

        # Store queue for later processing
        self.progress_queues[request_id] = progress_queue
        return persistent_progress_callback
    async def _store_progress_callback_state(self, agent_id: str, callback_func):
        """Store progress callback for reconnection scenarios."""
        self.persistent_callbacks[agent_id] = callback_func

    async def _restore_progress_callbacks(self):
        """Restore progress callbacks after reconnection."""
        for agent_id, callback_func in self.persistent_callbacks.items():
            agent = self.local_agents.get(agent_id)
            if agent and hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(callback_func)

    def on(self, event_name: str, handler: Callable[[dict], Awaitable[None]]):
        """Register an async callback function to handle a custom event from the server."""
        self.app.print(f"Handler for custom event '{event_name}' registered.")
        self.custom_event_handlers[event_name] = handler

    async def send_custom_event(self, event_name: str, data: dict[str, Any]):
        """Send a custom event with a JSON payload to the server."""
        if not self.is_connected or not self.ws or not self.ws.open:
            self.app.print("Cannot send custom event: Not connected.")
            return

        try:
            message = WsMessage(event=event_name, data=data)
            await self.ws.send(message.model_dump_json())
            self.app.print(f"Sent custom event '{event_name}' to server.")
        except Exception as e:
            self.app.print(f"Failed to send custom event: {e}")
            await self._handle_connection_error()

    async def _listen(self):
        """Robust message listening loop with immediate connection loss detection."""
        self.app.print("Registry client is now listening for incoming requests...")

        try:
            while self.is_connected and self.ws and self.ws.open:
                try:
                    # Check connection state before each recv attempt
                    if self.ws.closed:
                        self.app.print("WebSocket is closed - triggering reconnect")
                        break

                    message_raw = await asyncio.wait_for(self.ws.recv(), timeout=5.0)

                    # Handle different message types immediately
                    if isinstance(message_raw, bytes):
                        # Server ping - respond immediately
                        continue

                    # Process text messages
                    try:
                        message = WsMessage.model_validate_json(message_raw)
                        # Handle critical messages immediately, others in background
                        if message.event in ['agent_registered']:
                            await self._handle_message(message)
                        else:
                            # Handle non-critical messages in background to avoid blocking
                            task = asyncio.create_task(self._handle_message(message))
                            self.message_handler_tasks.add(task)
                            # Clean completed tasks
                            self.message_handler_tasks = {t for t in self.message_handler_tasks if not t.done()}

                    except Exception as e:
                        self.app.print(f"Error processing message: {e} | Raw: {message_raw[:200]}")

                except asyncio.TimeoutError:
                    # Normal timeout - check connection health
                    if not self.ws or not self.ws.open or self.ws.closed:
                        self.app.print("Connection health check failed during timeout")
                        break
                    continue

                except ConnectionClosed as e:
                    self.app.print(f"Connection closed by server: {e}")
                    break

                except Exception as e:
                    # Any other WebSocket error means connection is likely dead
                    if "ConnectionClosedError" in str(type(e)) or "IncompleteReadError" in str(type(e)):
                        self.app.print(f"Connection lost: {e}")
                        break
                    else:
                        self.app.print(f"Unexpected error in listen loop: {e}")
                        # Don't break on unexpected errors, but log them
                        await asyncio.sleep(0.1)

        except Exception as e:
            self.app.print(f"Fatal error in listen loop: {e}")
        finally:
            # Always trigger reconnection attempt
            if self.should_reconnect:
                asyncio.create_task(self._trigger_reconnect())

    async def _handle_message(self, message: WsMessage):
        """Handle incoming WebSocket messages with non-blocking execution."""
        try:
            if message.event == 'agent_registered':
                # Handle registration confirmation immediately
                reg_info = AgentRegistered.model_validate(message.data)
                reg_id = None
                for rid, future in self.pending_registrations.items():
                    if not future.done():
                        reg_id = rid
                        break

                if reg_id and reg_id in self.pending_registrations:
                    if not self.pending_registrations[reg_id].done():
                        self.pending_registrations[reg_id].set_result(reg_info)
                else:
                    self.app.print("Received agent_registered but no pending registration found")

            elif message.event == 'run_request':
                # Handle run requests in background - NEVER block here
                run_data = RunRequest.model_validate(message.data)
                asyncio.create_task(self._handle_run_request(run_data))

            elif message.event in self.custom_event_handlers:
                # Handle custom events in background
                self.app.print(f"Received custom event '{message.event}' from server.")
                handler = self.custom_event_handlers[message.event]
                asyncio.create_task(handler(message.data))

            else:
                self.app.print(f"Received unhandled event from server: '{message.event}'")

        except Exception as e:
            self.app.print(f"Error handling message: {e}")
            # Don't let message handling errors kill the connection

    async def register(self, agent_instance: Any, public_name: str, description: str | None = None) -> AgentRegistered | None:
        """Register an agent with the server."""
        if not self.is_connected or not self.ws:
            self.app.print("Not connected. Cannot register agent.")
            return None

        try:
            # Create registration request
            registration = AgentRegistration(public_name=public_name, description=description)
            message = WsMessage(event='register', data=registration.model_dump())

            # Create future for registration response
            reg_id = f"reg_{self.registration_counter}"
            self.registration_counter += 1
            self.pending_registrations[reg_id] = asyncio.Future()

            # Send registration request
            await self.ws.send(message.model_dump_json())
            self.app.print(f"Sent registration request for agent '{public_name}'")

            # Wait for registration confirmation
            try:
                reg_info = await asyncio.wait_for(self.pending_registrations[reg_id], timeout=30.0)

                # Store agent and registration info
                self.local_agents[reg_info.public_agent_id] = agent_instance
                self.registered_info[reg_info.public_agent_id] = reg_info

                self.app.print(f"Agent '{public_name}' registered successfully.")
                self.app.print(f"  Public URL: {reg_info.public_url}")
                self.app.print(f"  API Key: {reg_info.public_api_key}")

                return reg_info

            except TimeoutError:
                self.app.print("Timeout waiting for registration confirmation.")
                return None

        except Exception as e:
            self.app.print(f"Error during registration: {e}")
            return None
        finally:
            # Cleanup pending registration
            self.pending_registrations.pop(reg_id, None)

    async def _handle_run_request(self, run_request: RunRequest):
        """Handle run request - start agent in completely separate task."""
        agent_id = run_request.public_agent_id
        agent = self.local_agents.get(agent_id)

        if not agent:
            await self._stream_error(run_request.request_id, f"Agent with ID {agent_id} not found")
            return

        # Start agent execution in separate task - NEVER await here
        execution_task = asyncio.create_task(
            self._execute_agent_with_monitoring(agent, run_request)
        )

        # Store task but don't wait for it
        self.running_executions[run_request.request_id] = execution_task

        self.app.print(f"🚀 Agent execution started in background: {run_request.request_id}")
        # This method returns immediately - agent runs in background
    async def _execute_agent_with_monitoring(self, agent: Any, run_request: RunRequest):
        """Execute agent in completely separate task - never blocks main connection."""
        request_id = run_request.request_id
        agent_id = run_request.public_agent_id

        try:
            # Create progress streaming callback
            progress_callback = await self._create_streaming_progress_callback(request_id, agent_id)

            # Store original callback
            original_callback = getattr(agent, 'progress_callback', None)

            # Set streaming progress callback
            if hasattr(agent, 'set_progress_callback'):
                agent.set_progress_callback(progress_callback)
            elif hasattr(agent, 'progress_callback'):
                agent.progress_callback = progress_callback

            # Store for reconnection scenarios
            self.persistent_callbacks[agent_id] = progress_callback
            self.active_streams.add(request_id)

            self.app.print(f"🚀 Starting agent execution in separate task: {request_id}")

            # EXECUTE THE AGENT - this can run for hours/days
            final_result = await agent.a_run(
                query=run_request.query,
                session_id=run_request.session_id,
                **run_request.kwargs
            )

            # Send final result
            await self._stream_final_result(request_id, final_result, agent_id, run_request.session_id)

            self.app.print(f"✅ Agent execution completed: {request_id}")

        except Exception as e:
            self.app.print(f"❌ Agent execution failed: {e}")
            await self._stream_error(request_id, str(e))
            import traceback
            traceback.print_exc()

        finally:
            # Cleanup
            await self.running_executions.pop(request_id, None)
            self.persistent_callbacks.pop(agent_id, None)
            self.active_streams.discard(request_id)

            # Close progress queue
            if request_id in self.progress_queues:
                queue = self.progress_queues.pop(request_id)
                # Signal queue processor to stop for this request
                try:
                    await queue.put(None)  # Sentinel value
                except:
                    pass

            # Restore original callback
            try:
                if hasattr(agent, 'set_progress_callback'):
                    agent.set_progress_callback(original_callback)
                elif hasattr(agent, 'progress_callback'):
                    agent.progress_callback = original_callback
            except Exception as cleanup_error:
                self.app.print(f"Warning: Callback cleanup failed: {cleanup_error}")

    async def _stream_final_result(self, request_id: str, final_result: Any, agent_id: str, session_id: str):
        """Stream final result immediately."""
        final_event = ProgressEvent(
            event_type="execution_complete",
            node_name="RegistryClient",
            success=True,
            metadata={
                "result": final_result,
                "agent_id": agent_id,
                "session_id": session_id
            }
        )

        final_message = ExecutionResult(
            request_id=request_id,
            payload=final_event.to_dict(),
            is_final=True
        )

        # Stream final result with high priority
        max_attempts = 10
        for attempt in range(max_attempts):
            try:
                if await self._check_connection_health():
                    success = await self._send_message('execution_result', final_message.model_dump())
                    if success:
                        self.app.print(f"✅ Final result streamed successfully")
                        return

                await asyncio.sleep(1.0 * (attempt + 1))  # Longer delays for final result

            except Exception as e:
                self.app.print(f"Final result stream attempt {attempt + 1} failed: {e}")

        self.app.print(f"❌ Failed to stream final result after {max_attempts} attempts")

    async def _stream_error(self, request_id: str, error_message: str):
        """Stream error immediately."""
        error_payload = ExecutionError(request_id=request_id, error=error_message)

        for attempt in range(5):
            try:
                if await self._check_connection_health():
                    success = await self._send_message('execution_error', error_payload.model_dump())
                    if success:
                        return
                await asyncio.sleep(0.5 * (attempt + 1))
            except Exception as e:
                self.app.print(f"Error stream attempt {attempt + 1} failed: {e}")

    async def _create_streaming_progress_callback(self, request_id: str, agent_id: str):
        """Create callback that streams progress immediately as it comes."""
        # Create queue for this specific request
        progress_queue = asyncio.Queue()
        self.progress_queues[request_id] = progress_queue

        # Start dedicated processor for this request
        processor_task = asyncio.create_task(
            self._process_progress_stream(request_id, progress_queue)
        )

        async def streaming_progress_callback(event: ProgressEvent):
            """Stream progress immediately - no batching, no delays."""
            try:
                if request_id in self.active_streams:
                    # Put in queue for immediate processing
                    await progress_queue.put(event)
            except Exception as e:
                self.app.print(f"Progress streaming error: {e}")

        return streaming_progress_callback

    async def _process_progress_stream(self, request_id: str, progress_queue: asyncio.Queue):
        """Process progress stream in real-time - separate task per request."""
        self.app.print(f"📡 Started progress streaming for request: {request_id}")

        while request_id in self.active_streams:
            try:
                # Get next progress event (blocking)
                event = await progress_queue.get()

                # Sentinel value to stop
                if event is None:
                    break

                # Stream immediately - no batching
                await self._stream_progress_immediately(request_id, event)

            except Exception as e:
                self.app.print(f"Progress stream processing error: {e}")
                await asyncio.sleep(0.1)  # Brief pause on error

        self.app.print(f"📡 Stopped progress streaming for request: {request_id}")

    async def _stream_progress_immediately(self, request_id: str, event: ProgressEvent):
        """Stream single progress event immediately."""
        max_attempts = 3

        for attempt in range(max_attempts):
            try:
                if await self._check_connection_health():
                    result = ExecutionResult(
                        request_id=request_id,
                        payload=event.to_dict(),
                        is_final=False
                    )

                    success = await self._send_message('execution_result', result.model_dump())
                    if success:
                        return  # Successfully streamed

                # Connection unhealthy - brief wait before retry
                await asyncio.sleep(0.2 * (attempt + 1))

            except Exception as e:
                self.app.print(f"Stream attempt {attempt + 1} failed: {e}")
                if attempt < max_attempts - 1:
                    await asyncio.sleep(0.2 * (attempt + 1))

        # All attempts failed - but don't crash, just log
        self.app.print(f"⚠️ Failed to stream progress after {max_attempts} attempts")


    async def send_ui_progress(self, progress_data: dict[str, Any], retry_count: int = 3):
        """Enhanced UI progress sender with retry logic."""
        if not self.is_connected or not self.ws or not self.ws.open:
            self.app.print("Registry client WebSocket not connected - queuing progress update")
            # Could implement a queue here for offline progress updates
            return False

        for attempt in range(retry_count):
            try:
                # Structure progress message for registry server
                ui_message = {
                    "timestamp": progress_data.get('timestamp', asyncio.get_event_loop().time()),
                    "agent_id": progress_data.get('agent_id', 'unknown'),
                    "event_type": progress_data.get('event_type', 'unknown'),
                    "status": progress_data.get('status', 'processing'),
                    "agent_name": progress_data.get('agent_name', 'Unknown'),
                    "node_name": progress_data.get('node_name', 'Unknown'),
                    "session_id": progress_data.get('session_id'),
                    "metadata": progress_data.get('metadata', {}),

                    # Enhanced progress data for UI panels
                    "outline_progress": progress_data.get('progress_data', {}).get('outline', {}),
                    "activity_info": progress_data.get('progress_data', {}).get('activity', {}),
                    "meta_tool_info": progress_data.get('progress_data', {}).get('meta_tool', {}),
                    "system_status": progress_data.get('progress_data', {}).get('system', {}),
                    "graph_info": progress_data.get('progress_data', {}).get('graph', {}),

                    # UI flags for selective updates
                    "ui_flags": progress_data.get('ui_flags', {}),

                    # Performance metrics
                    "performance": progress_data.get('performance', {}),

                    # Message metadata
                    "message_id": f"msg_{asyncio.get_event_loop().time()}_{attempt}",
                    "retry_count": attempt
                }

                # Send as WsMessage
                message = WsMessage(event='ui_progress_update', data=ui_message)
                await self.ws.send(message.model_dump_json())

                # Success - break retry loop
                self.app.print(
                    f"📤 Sent UI progress: {progress_data.get('event_type')} | {progress_data.get('status')} (attempt {attempt + 1})")
                return True

            except Exception as e:
                self.app.print(f"Failed to send UI progress (attempt {attempt + 1}/{retry_count}): {e}")
                if attempt < retry_count - 1:
                    await asyncio.sleep(0.5 * (attempt + 1))  # Exponential backoff
                else:
                    await self._handle_connection_error()
                    return False

        return False

    async def send_agent_status(self, agent_id: str, status: str, details: dict[str, Any] = None):
        """Send agent status updates."""
        if not self.is_connected or not self.ws or not self.ws.open:
            return

        try:
            status_message = {
                "agent_id": agent_id,
                "status": status,
                "details": details or {},
                "timestamp": asyncio.get_event_loop().time(),
                "capabilities": ["chat", "progress_tracking", "outline_visualization", "meta_tool_monitoring"]
            }

            message = WsMessage(event='agent_status_update', data=status_message)
            await self.ws.send(message.model_dump_json())

        except Exception as e:
            self.app.print(f"Failed to send agent status: {e}")
            await self._handle_connection_error()

    async def _send_error(self, request_id: str, error_message: str):
        """Send error message to server."""
        error_payload = ExecutionError(request_id=request_id, error=error_message)
        await self._send_message('execution_error', error_payload.model_dump())

    async def _check_connection_health(self) -> bool:
        """Check if the WebSocket connection is actually healthy."""
        if not self.ws:
            return False

        try:
            # Check basic connection state
            if self.ws.closed or not self.ws.open:
                return False

            # Try a quick ping to verify connectivity
            pong_waiter = await self.ws.ping()
            await asyncio.wait_for(pong_waiter, timeout=3.0)
            return True

        except Exception as e:
            self.app.print(f"Connection health check failed: {e}")
            return False

    async def _send_message(self, event: str, data: dict, max_retries: int = 3):
        """Enhanced message sending with connection health verification."""
        for attempt in range(max_retries):
            # Check connection health before attempting to send
            if not await self._check_connection_health():
                self.app.print(f"Connection unhealthy for message '{event}' (attempt {attempt + 1})")

                if attempt < max_retries - 1:
                    await asyncio.sleep(0.5 * (attempt + 1))
                    continue
                else:
                    self.app.print(f"Cannot send message '{event}': Connection permanently failed")
                    asyncio.create_task(self._trigger_reconnect())
                    return False

            try:
                message = WsMessage(event=event, data=data)
                await self.ws.send(message.model_dump_json())
                return True

            except Exception as e:
                self.app.print(f"Send attempt {attempt + 1} failed for '{event}': {e}")

                # Check if this is a connection-related error
                error_str = str(e).lower()
                if any(err in error_str for err in ['connectionclosed', 'incomplete', 'connection', 'closed']):
                    self.app.print("Connection error detected - triggering reconnect")
                    asyncio.create_task(self._trigger_reconnect())
                    return False

                if attempt < max_retries - 1:
                    await asyncio.sleep(0.5 * (attempt + 1))

        return False
    async def _send_final_result_with_retry(self, request_id: str, final_result: Any, agent_id: str, session_id: str):
        """Send final result with robust retry logic."""
        final_event = ProgressEvent(
            event_type="execution_complete",
            node_name="RegistryClient",
            success=True,
            metadata={
                "result": final_result,
                "agent_id": agent_id,
                "session_id": session_id
            }
        )

        final_message = ExecutionResult(
            request_id=request_id,
            payload=final_event.to_dict(),
            is_final=True
        )

        max_retries = 10
        base_delay = 2

        for attempt in range(max_retries):
            try:
                if not self.is_connected or not self.ws or not self.ws.open:
                    self.app.print(f"⚠️  Connection lost - waiting for reconnection (attempt {attempt + 1})")
                    await asyncio.sleep(base_delay * (attempt + 1))
                    continue

                await self._send_message('execution_result', final_message.model_dump())
                self.app.print(f"✅ Final result sent successfully on attempt {attempt + 1}")
                return

            except Exception as e:
                delay = base_delay * (2 ** attempt)
                self.app.print(f"❌ Failed to send final result (attempt {attempt + 1}): {e}")
                if attempt < max_retries - 1:
                    await asyncio.sleep(delay)

        self.app.print(f"❌ Failed to send final result after {max_retries} attempts")

    async def _send_error_with_retry(self, request_id: str, error_message: str):
        """Send error message with retry logic."""
        max_retries = 5

        for attempt in range(max_retries):
            try:
                if self.is_connected and self.ws and self.ws.open:
                    await self._send_error(request_id, error_message)
                    return
                else:
                    await asyncio.sleep(2 * (attempt + 1))
            except Exception as e:
                self.app.print(f"Error sending error message (attempt {attempt + 1}): {e}")
                if attempt < max_retries - 1:
                    await asyncio.sleep(2 * (attempt + 1))

    async def _handle_connection_error(self):
        """Handle connection errors and cleanup."""
        self.is_connected = False
        if self.ws:
            with contextlib.suppress(builtins.BaseException):
                await self.ws.close()
            self.ws = None

    async def disconnect(self):
        """Enhanced disconnect with complete task cleanup."""
        self.app.print("Initiating clean shutdown...")
        self.is_connected = False
        self.should_reconnect = False

        # Cancel all background tasks
        tasks_to_cancel = []

        if self.connection_task and not self.connection_task.done():
            tasks_to_cancel.append(self.connection_task)

        if self.ping_task and not self.ping_task.done():
            tasks_to_cancel.append(self.ping_task)

        # Cancel message handler tasks
        for task in list(self.message_handler_tasks):
            if not task.done():
                tasks_to_cancel.append(task)

        # Cancel running executions
        for task in list(self.running_executions.values()):
            if not task.done():
                tasks_to_cancel.append(task)

        if tasks_to_cancel:
            self.app.print(f"Cancelling {len(tasks_to_cancel)} background tasks...")
            for task in tasks_to_cancel:
                task.cancel()

            # Wait for cancellation with timeout
            try:
                await asyncio.wait_for(
                    asyncio.gather(*tasks_to_cancel, return_exceptions=True),
                    timeout=5.0
                )
            except asyncio.TimeoutError:
                self.app.print("Warning: Some tasks didn't cancel within timeout")

        # Close WebSocket connection
        if self.ws:
            with contextlib.suppress(Exception):
                await self.ws.close()
            self.ws = None

        # Cancel pending registrations
        for future in self.pending_registrations.values():
            if not future.done():
                future.cancel()
        self.pending_registrations.clear()

        # Clear state
        self.message_handler_tasks.clear()
        self.running_executions.clear()
        self.persistent_callbacks.clear()

        self.connection_task = None
        self.ping_task = None

        self.app.print("✅ Registry client shutdown completed")
cancel_execution(request_id) async

Cancel a running execution.

Source code in toolboxv2/mods/registry/client.py
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
async def cancel_execution(self, request_id: str) -> bool:
    """Cancel a running execution."""
    try:
        if request_id not in self.running_executions:
            self.app.print(f"❌ Execution {request_id} not found")
            return False

        execution_task = self.running_executions[request_id]

        if execution_task.done():
            self.app.print(f"⚠️  Execution {request_id} already completed")
            return True

        # Cancel the task
        execution_task.cancel()

        try:
            # Wait a moment for graceful cancellation
            await asyncio.wait_for(execution_task, timeout=5.0)
        except asyncio.CancelledError:
            self.app.print(f"✅ Execution {request_id} cancelled successfully")
        except asyncio.TimeoutError:
            self.app.print(f"⚠️  Execution {request_id} cancellation timeout - may still be running")
        except Exception as e:
            self.app.print(f"⚠️  Execution {request_id} cancellation resulted in exception: {e}")

        # Send cancellation notice to server
        try:
            if self.is_connected and self.ws and self.ws.open:
                cancellation_event = ProgressEvent(
                    event_type="execution_cancelled",
                    node_name="RegistryClient",
                    success=False,
                    metadata={
                        "request_id": request_id,
                        "cancellation_reason": "client_requested",
                        "timestamp": asyncio.get_event_loop().time()
                    }
                )

                cancellation_message = ExecutionResult(
                    request_id=request_id,
                    payload=cancellation_event.to_dict(),
                    is_final=True
                )

                await self._send_message('execution_result', cancellation_message.model_dump())

        except Exception as e:
            self.app.print(f"Failed to send cancellation notice to server: {e}")

        # Cleanup
        self.running_executions.pop(request_id, None)

        return True

    except Exception as e:
        self.app.print(f"Error cancelling execution {request_id}: {e}")
        return False
cleanup_completed_executions() async

Clean up completed execution tasks.

Source code in toolboxv2/mods/registry/client.py
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
async def cleanup_completed_executions(self):
    """Clean up completed execution tasks."""
    try:
        completed_tasks = []

        for request_id, task in self.running_executions.items():
            if task.done():
                completed_tasks.append(request_id)

        for request_id in completed_tasks:
            self.running_executions.pop(request_id, None)
            self.app.print(f"🧹 Cleaned up completed execution: {request_id}")

        return len(completed_tasks)

    except Exception as e:
        self.app.print(f"Error during cleanup: {e}")
        return 0
connect(server_url, timeout=30.0) async

Connect and start all background tasks.

Source code in toolboxv2/mods/registry/client.py
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
async def connect(self, server_url: str, timeout: float = 30.0):
    """Connect and start all background tasks."""
    if not ws_client:
        self.app.print("Websockets library not installed. Please run 'pip install websockets'")
        return False

    if self.ws and self.ws.open:
        self.app.print("Already connected to the registry server.")
        return True

    self.server_url = server_url
    self.should_reconnect = True
    self.reconnect_in_progress = False

    try:
        self.app.print(f"Connecting to Registry Server at {server_url}...")
        self.ws = await asyncio.wait_for(
            ws_client.connect(server_url),
            timeout=timeout
        )

        self.is_connected = True
        self.reconnect_attempts = 0

        # Start all background tasks
        await self._start_all_background_tasks()

        self.app.print(f"✅ Successfully connected and started all tasks")
        return True

    except asyncio.TimeoutError:
        self.app.print(f"❌ Connection timeout after {timeout}s")
        return False
    except Exception as e:
        self.app.print(f"❌ Connection failed: {e}")
        return False
disconnect() async

Enhanced disconnect with complete task cleanup.

Source code in toolboxv2/mods/registry/client.py
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
async def disconnect(self):
    """Enhanced disconnect with complete task cleanup."""
    self.app.print("Initiating clean shutdown...")
    self.is_connected = False
    self.should_reconnect = False

    # Cancel all background tasks
    tasks_to_cancel = []

    if self.connection_task and not self.connection_task.done():
        tasks_to_cancel.append(self.connection_task)

    if self.ping_task and not self.ping_task.done():
        tasks_to_cancel.append(self.ping_task)

    # Cancel message handler tasks
    for task in list(self.message_handler_tasks):
        if not task.done():
            tasks_to_cancel.append(task)

    # Cancel running executions
    for task in list(self.running_executions.values()):
        if not task.done():
            tasks_to_cancel.append(task)

    if tasks_to_cancel:
        self.app.print(f"Cancelling {len(tasks_to_cancel)} background tasks...")
        for task in tasks_to_cancel:
            task.cancel()

        # Wait for cancellation with timeout
        try:
            await asyncio.wait_for(
                asyncio.gather(*tasks_to_cancel, return_exceptions=True),
                timeout=5.0
            )
        except asyncio.TimeoutError:
            self.app.print("Warning: Some tasks didn't cancel within timeout")

    # Close WebSocket connection
    if self.ws:
        with contextlib.suppress(Exception):
            await self.ws.close()
        self.ws = None

    # Cancel pending registrations
    for future in self.pending_registrations.values():
        if not future.done():
            future.cancel()
    self.pending_registrations.clear()

    # Clear state
    self.message_handler_tasks.clear()
    self.running_executions.clear()
    self.persistent_callbacks.clear()

    self.connection_task = None
    self.ping_task = None

    self.app.print("✅ Registry client shutdown completed")
get_connection_status() async

Get detailed connection status information.

Source code in toolboxv2/mods/registry/client.py
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
async def get_connection_status(self) -> dict[str, Any]:
    """Get detailed connection status information."""
    try:
        connection_status = {
            "is_connected": self.is_connected,
            "server_url": self.server_url,
            "reconnect_attempts": self.reconnect_attempts,
            "max_reconnect_attempts": self.max_reconnect_attempts,
            "should_reconnect": self.should_reconnect,
            "reconnect_in_progress": self.reconnect_in_progress,
            "websocket_state": None,
            "websocket_open": False,
            "tasks": {
                "connection_task_running": self.connection_task and not self.connection_task.done(),
                "ping_task_running": self.ping_task and not self.ping_task.done(),
            },
            "registered_agents_count": len(self.local_agents),
            "running_executions_count": len(self.running_executions),
            "pending_registrations_count": len(self.pending_registrations),
            "persistent_callbacks_count": len(self.persistent_callbacks),
            "last_ping_time": getattr(self, 'last_ping_time', None),
            "connection_uptime": None,
            "connection_established_at": getattr(self, 'connection_established_at', None),
        }

        # WebSocket specific status
        if self.ws:
            connection_status.update({
                "websocket_state": str(self.ws.state.name) if hasattr(self.ws.state, 'name') else str(
                    self.ws.state),
                "websocket_open": self.ws.open,
                "websocket_closed": self.ws.closed,
            })

        # Calculate uptime
        if hasattr(self, 'connection_established_at') and self.connection_established_at:
            connection_status[
                "connection_uptime"] = asyncio.get_event_loop().time() - self.connection_established_at

        return connection_status

    except Exception as e:
        self.app.print(f"Error getting connection status: {e}")
        return {
            "error": str(e),
            "is_connected": False,
            "server_url": self.server_url,
        }
get_diagnostics() async

Get comprehensive diagnostic information.

Source code in toolboxv2/mods/registry/client.py
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
async def get_diagnostics(self) -> dict[str, Any]:
    """Get comprehensive diagnostic information."""
    try:
        diagnostics = {
            "connection_status": await self.get_connection_status(),
            "registered_agents": await self.get_registered_agents(),
            "running_executions": await self.get_running_executions(),
            "health_status": await self.health_check(),
            "system_info": {
                "python_version": sys.version,
                "asyncio_running": True,
                "event_loop": str(asyncio.get_running_loop()),
                "thread_name": threading.current_thread().name,
            },
            "performance_metrics": {
                "total_messages_sent": getattr(self, 'total_messages_sent', 0),
                "total_messages_received": getattr(self, 'total_messages_received', 0),
                "total_reconnections": self.reconnect_attempts,
                "total_registrations": len(self.registered_info),
                "memory_usage": self._get_memory_usage(),
            },
            "error_log": getattr(self, 'recent_errors', []),
        }

        return diagnostics

    except Exception as e:
        return {
            "diagnostics_error": str(e),
            "timestamp": asyncio.get_event_loop().time()
        }
get_registered_agents() async

Get all registered agents information.

Source code in toolboxv2/mods/registry/client.py
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
async def get_registered_agents(self) -> dict[str, AgentRegistered]:
    """Get all registered agents information."""
    try:
        agents_info = {}

        for agent_id, reg_info in self.registered_info.items():
            # Get agent instance if available
            agent_instance = self.local_agents.get(agent_id)

            # Create enhanced agent info
            agent_data = {
                "registration_info": reg_info,
                "agent_available": agent_instance is not None,
                "agent_type": type(agent_instance).__name__ if agent_instance else "Unknown",
                "has_progress_callback": hasattr(agent_instance, 'progress_callback') if agent_instance else False,
                "supports_progress_callback": hasattr(agent_instance,
                                                      'set_progress_callback') if agent_instance else False,
                "is_persistent_callback_active": agent_id in self.persistent_callbacks,
                "registration_timestamp": getattr(reg_info, 'registration_timestamp', None),
            }

            # Add agent capabilities if available
            if agent_instance and hasattr(agent_instance, 'get_capabilities'):
                try:
                    agent_data["capabilities"] = await agent_instance.get_capabilities()
                except Exception as e:
                    agent_data["capabilities_error"] = str(e)

            agents_info[agent_id] = agent_data

        return agents_info

    except Exception as e:
        self.app.print(f"Error getting registered agents: {e}")
        return {}
get_running_executions() async

Get information about currently running executions.

Source code in toolboxv2/mods/registry/client.py
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
async def get_running_executions(self) -> dict[str, dict[str, Any]]:
    """Get information about currently running executions."""
    try:
        executions_info = {}

        for request_id, execution_task in self.running_executions.items():
            execution_info = {
                "request_id": request_id,
                "task_done": execution_task.done(),
                "task_cancelled": execution_task.cancelled(),
                "start_time": getattr(execution_task, 'start_time', None),
                "running_time": None,
                "task_exception": None,
                "task_result": None,
            }

            # Calculate running time
            if hasattr(execution_task, 'start_time') and execution_task.start_time:
                execution_info["running_time"] = asyncio.get_event_loop().time() - execution_task.start_time

            # Get task status details
            if execution_task.done():
                try:
                    if execution_task.exception():
                        execution_info["task_exception"] = str(execution_task.exception())
                    else:
                        execution_info["task_result"] = "completed_successfully"
                except Exception as e:
                    execution_info["task_status_error"] = str(e)

            executions_info[request_id] = execution_info

        return executions_info

    except Exception as e:
        self.app.print(f"Error getting running executions: {e}")
        return {}
health_check() async

Perform a health check of the connection.

Source code in toolboxv2/mods/registry/client.py
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
async def health_check(self) -> bool:
    """Perform a health check of the connection."""
    try:
        # Basic connection checks
        if not self.is_connected:
            self.app.print("🔍 Health check: Not connected")
            return False

        if not self.ws or not self.ws.open:
            self.app.print("🔍 Health check: WebSocket not open")
            return False

        # Ping test
        try:
            pong_waiter = await self.ws.ping()
            await asyncio.wait_for(pong_waiter, timeout=10.0)

            # Update last ping time
            self.last_ping_time = asyncio.get_event_loop().time()

            # Test message sending
            test_message = WsMessage(
                event='health_check',
                data={
                    "timestamp": self.last_ping_time,
                    "client_id": getattr(self, 'client_id', 'unknown'),
                    "registered_agents": list(self.local_agents.keys()),
                    "running_executions": list(self.running_executions.keys())
                }
            )

            await self.ws.send(test_message.model_dump_json())

            self.app.print("✅ Health check: Connection healthy")
            return True

        except asyncio.TimeoutError:
            self.app.print("❌ Health check: Ping timeout")
            return False
        except Exception as ping_error:
            self.app.print(f"❌ Health check: Ping failed - {ping_error}")
            return False

    except Exception as e:
        self.app.print(f"❌ Health check: Error - {e}")
        return False
on(event_name, handler)

Register an async callback function to handle a custom event from the server.

Source code in toolboxv2/mods/registry/client.py
627
628
629
630
def on(self, event_name: str, handler: Callable[[dict], Awaitable[None]]):
    """Register an async callback function to handle a custom event from the server."""
    self.app.print(f"Handler for custom event '{event_name}' registered.")
    self.custom_event_handlers[event_name] = handler
register(agent_instance, public_name, description=None) async

Register an agent with the server.

Source code in toolboxv2/mods/registry/client.py
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
async def register(self, agent_instance: Any, public_name: str, description: str | None = None) -> AgentRegistered | None:
    """Register an agent with the server."""
    if not self.is_connected or not self.ws:
        self.app.print("Not connected. Cannot register agent.")
        return None

    try:
        # Create registration request
        registration = AgentRegistration(public_name=public_name, description=description)
        message = WsMessage(event='register', data=registration.model_dump())

        # Create future for registration response
        reg_id = f"reg_{self.registration_counter}"
        self.registration_counter += 1
        self.pending_registrations[reg_id] = asyncio.Future()

        # Send registration request
        await self.ws.send(message.model_dump_json())
        self.app.print(f"Sent registration request for agent '{public_name}'")

        # Wait for registration confirmation
        try:
            reg_info = await asyncio.wait_for(self.pending_registrations[reg_id], timeout=30.0)

            # Store agent and registration info
            self.local_agents[reg_info.public_agent_id] = agent_instance
            self.registered_info[reg_info.public_agent_id] = reg_info

            self.app.print(f"Agent '{public_name}' registered successfully.")
            self.app.print(f"  Public URL: {reg_info.public_url}")
            self.app.print(f"  API Key: {reg_info.public_api_key}")

            return reg_info

        except TimeoutError:
            self.app.print("Timeout waiting for registration confirmation.")
            return None

    except Exception as e:
        self.app.print(f"Error during registration: {e}")
        return None
    finally:
        # Cleanup pending registration
        self.pending_registrations.pop(reg_id, None)
send_agent_status(agent_id, status, details=None) async

Send agent status updates.

Source code in toolboxv2/mods/registry/client.py
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
async def send_agent_status(self, agent_id: str, status: str, details: dict[str, Any] = None):
    """Send agent status updates."""
    if not self.is_connected or not self.ws or not self.ws.open:
        return

    try:
        status_message = {
            "agent_id": agent_id,
            "status": status,
            "details": details or {},
            "timestamp": asyncio.get_event_loop().time(),
            "capabilities": ["chat", "progress_tracking", "outline_visualization", "meta_tool_monitoring"]
        }

        message = WsMessage(event='agent_status_update', data=status_message)
        await self.ws.send(message.model_dump_json())

    except Exception as e:
        self.app.print(f"Failed to send agent status: {e}")
        await self._handle_connection_error()
send_custom_event(event_name, data) async

Send a custom event with a JSON payload to the server.

Source code in toolboxv2/mods/registry/client.py
632
633
634
635
636
637
638
639
640
641
642
643
644
async def send_custom_event(self, event_name: str, data: dict[str, Any]):
    """Send a custom event with a JSON payload to the server."""
    if not self.is_connected or not self.ws or not self.ws.open:
        self.app.print("Cannot send custom event: Not connected.")
        return

    try:
        message = WsMessage(event=event_name, data=data)
        await self.ws.send(message.model_dump_json())
        self.app.print(f"Sent custom event '{event_name}' to server.")
    except Exception as e:
        self.app.print(f"Failed to send custom event: {e}")
        await self._handle_connection_error()
send_ui_progress(progress_data, retry_count=3) async

Enhanced UI progress sender with retry logic.

Source code in toolboxv2/mods/registry/client.py
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
async def send_ui_progress(self, progress_data: dict[str, Any], retry_count: int = 3):
    """Enhanced UI progress sender with retry logic."""
    if not self.is_connected or not self.ws or not self.ws.open:
        self.app.print("Registry client WebSocket not connected - queuing progress update")
        # Could implement a queue here for offline progress updates
        return False

    for attempt in range(retry_count):
        try:
            # Structure progress message for registry server
            ui_message = {
                "timestamp": progress_data.get('timestamp', asyncio.get_event_loop().time()),
                "agent_id": progress_data.get('agent_id', 'unknown'),
                "event_type": progress_data.get('event_type', 'unknown'),
                "status": progress_data.get('status', 'processing'),
                "agent_name": progress_data.get('agent_name', 'Unknown'),
                "node_name": progress_data.get('node_name', 'Unknown'),
                "session_id": progress_data.get('session_id'),
                "metadata": progress_data.get('metadata', {}),

                # Enhanced progress data for UI panels
                "outline_progress": progress_data.get('progress_data', {}).get('outline', {}),
                "activity_info": progress_data.get('progress_data', {}).get('activity', {}),
                "meta_tool_info": progress_data.get('progress_data', {}).get('meta_tool', {}),
                "system_status": progress_data.get('progress_data', {}).get('system', {}),
                "graph_info": progress_data.get('progress_data', {}).get('graph', {}),

                # UI flags for selective updates
                "ui_flags": progress_data.get('ui_flags', {}),

                # Performance metrics
                "performance": progress_data.get('performance', {}),

                # Message metadata
                "message_id": f"msg_{asyncio.get_event_loop().time()}_{attempt}",
                "retry_count": attempt
            }

            # Send as WsMessage
            message = WsMessage(event='ui_progress_update', data=ui_message)
            await self.ws.send(message.model_dump_json())

            # Success - break retry loop
            self.app.print(
                f"📤 Sent UI progress: {progress_data.get('event_type')} | {progress_data.get('status')} (attempt {attempt + 1})")
            return True

        except Exception as e:
            self.app.print(f"Failed to send UI progress (attempt {attempt + 1}/{retry_count}): {e}")
            if attempt < retry_count - 1:
                await asyncio.sleep(0.5 * (attempt + 1))  # Exponential backoff
            else:
                await self._handle_connection_error()
                return False

    return False
get_registry_client(app)

Factory function to get a singleton RegistryClient instance.

Source code in toolboxv2/mods/registry/client.py
1266
1267
1268
1269
1270
1271
def get_registry_client(app: App) -> RegistryClient:
    """Factory function to get a singleton RegistryClient instance."""
    app_id = app.id
    if app_id not in registry_clients:
        registry_clients[app_id] = RegistryClient(app)
    return registry_clients[app_id]

demo_custom_messaging

setup_chain_with_live_updates() async

Example 3: Create agent chain with live progress broadcasting

Source code in toolboxv2/mods/registry/demo_custom_messaging.py
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
async def setup_chain_with_live_updates():
    """Example 3: Create agent chain with live progress broadcasting"""
    app = get_app("ChainLiveExample")
    isaa = app.get_mod("isaa")

    # Initialize ISAA
    await isaa.init_isaa()

    # Create and register specialized agents

    # Research agent
    researcher_builder = isaa.get_agent_builder("researcher_agent")
    researcher_builder.with_system_message(
        "You are a research specialist. Gather comprehensive information and provide detailed analysis. "
        "Always report your progress clearly."
    )
    #researcher_builder.with_models(complex_llm_model="openrouter/openai/gpt-4o")
    await isaa.register_agent(researcher_builder)

    # Writer agent
    writer_builder = isaa.get_agent_builder("writer_agent")
    writer_builder.with_system_message(
        "You are a professional writer. Create well-structured, engaging content from research data. "
        "Report your writing progress step by step."
    )
    #writer_builder.with_models(complex_llm_model="openrouter/openai/gpt-4o")
    await isaa.register_agent(writer_builder)

    # Reviewer agent
    reviewer_builder = isaa.get_agent_builder("reviewer_agent")
    reviewer_builder.with_system_message(
        "You are a quality reviewer. Check for accuracy, completeness, and suggest improvements. "
        "Report your review progress clearly."
    )
    # reviewer_builder.with_models(fast_llm_model="openrouter/anthropic/claude-3-haiku")
    await isaa.register_agent(reviewer_builder)

    # Get agent instances
    researcher = await isaa.get_agent("researcher_agent")
    writer = await isaa.get_agent("writer_agent")
    reviewer = await isaa.get_agent("reviewer_agent")

    # Create chain using the >> operator for sequential execution
    from pydantic import BaseModel
    class Topick(BaseModel):
        topic: str

    class MiniBlog(BaseModel):
        title: str
        content: str

    class Review(BaseModel):
        feedback: str
        better_title: str
        better_content: str

    chain = researcher >> CF(Topick) >> writer >> CF(MiniBlog) >> reviewer >> CF(Review)
    chain.name = "content_creation_chain"

    # Publish chain with live updates - Progress Callback wird automatisch eingerichtet
    result = await isaa.publish_and_host_agent(
        agent=chain,
        public_name="Content Creation Pipeline",
        description="Multi-agent chain with live progress: Research → Write → Review",
        registry_server="ws://localhost:8080/ws/registry/connect",
    )

    if result.get('public_url'):
        app.print("🔗 Chain published successfully with Live Progress UI!")
        app.print(f"   Local UI: {result['ui_url']}")
        app.print(f"   WebSocket: {result.get('registry_server')}")
        app.print(f"   WebSocket: {result.get('websocket_url')}")
        app.print(f"   Public URL: {result.get('public_url')}")
        app.print(f"   API Key: {result.get('public_api_key')}")
        print(result)

        # Example usage - test the chain with live updates
        #pp.print("\n🧪 Testing chain execution with live progress tracking:")
        #ry:
        #   result_text = await chain.a_run(
        #       query="Create a comprehensive article about renewable energy trends in 2024",
        #       session_id="demo-session"
        #   )
        #   app.print(f"✅ Chain completed successfully!")
        #   app.print(f"   Result length: {len(result_text)} characters")
        #   app.print("   All progress was tracked live in the UI!")
        #xcept Exception as e:
        #   app.print(f"❌ Chain execution failed: {e}")

        # Keep services running with live status
        try:
            while True:
                await asyncio.sleep(30)
                app.print("💓 Chain services live - ready for requests")
        except KeyboardInterrupt:
            app.print("Shutting down chain services...")
    else:
        app.print("❌ Failed to publish chain to registry")

    # Clean shutdown
    await researcher.close()
    await writer.close()
    await reviewer.close()
setup_complete_agent_system(local=False) async

Vollständiges Beispiel für Agent-System mit Live-Progress.

Source code in toolboxv2/mods/registry/demo_custom_messaging.py
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
async def setup_complete_agent_system(local=False):
    """Vollständiges Beispiel für Agent-System mit Live-Progress."""

    app = get_app("CompleteAgentSystem")
    isaa = app.get_mod("isaa")

    # ISAA initialisieren
    await isaa.init_isaa()

    # Erweiterten Agent erstellen
    advanced_builder = isaa.get_agent_builder("production_assistant")
    advanced_builder.with_system_message("""
        Du bist ein produktions-fertiger AI-Assistent mit detailliertem Progress-Tracking.

        Arbeitsweise:
        1. Analysiere die Anfrage sorgfältig
        2. Erstelle einen strukturierten Plan (Outline)
        3. Führe jeden Schritt methodisch aus
        4. Verwende Meta-Tools für komplexe Aufgaben
        5. Berichte kontinuierlich über deinen Fortschritt
        6. Liefere umfassende, gut strukturierte Antworten

        Zeige immer, welche Tools du verwendest und warum.
        Erkläre deine Reasoning-Loops transparent.
        """)

    # Agent registrieren
    await isaa.register_agent(advanced_builder)
    agent = await isaa.get_agent("production_assistant")

    # **Produktionsfertige Publish & Host - Ein Aufruf macht alles**
    result = await isaa.publish_and_host_agent(
        agent=agent,
        public_name="Production AI Assistant",
        registry_server="ws://localhost:8080/ws/registry/connect" if local else "wss://simplecore.app/ws/registry/connect",
        description="Production-ready AI assistant with comprehensive progress tracking, step-by-step reasoning, and meta-tool visualization. Supports real-time progress updates, outline tracking, and multi-user access.",
        access_level="public"
    )

    if result.get('success'):
        app.print("🎉 AGENT SYSTEM FULLY DEPLOYED!")
        app.print("")
        app.print("🌐 Public Access:")
        app.print(f"   URL: {result['public_url']}")
        app.print(f"   API Key: {result['public_api_key']}")
        app.print("")
        app.print("🖥️  Live UI:")
        app.print(f"   Registry UI: {result['ui_url']}")
        if result.get('local_ui'):
            app.print(f"   Local UI: {result['local_ui'].get('ui_url')}")
        app.print("")
        app.print("🔌 WebSocket:")
        app.print(f"   Live Updates: {result['websocket_url']}")
        app.print("")
        app.print("📋 cURL Test:")
        app.print(f"""curl -X POST {result['public_url']} \\
  -H "Content-Type: application/json" \\
  -H "Authorization: Bearer {result['public_api_key']}" \\
  -d '{{"query": "Create a detailed analysis of quantum computing with step-by-step progress", "session_id": "test-session"}}'""")

        # Lokaler Test des Agents
        app.print("\n🧪 Testing agent locally...")
        #await asyncio.sleep(5)
        #test_result = await agent.a_run(
        #    "hey",
        #    session_id="local_test"
        #)
        app.print("✅ Test completed successfully!")

        # Service am Leben halten
        try:
            while True:
                await asyncio.sleep(30)
                app.print("💓 Agent services running - ready for requests")
        except KeyboardInterrupt:
            app.print("🛑 Shutting down agent services...")
    else:
        app.print(f"❌ Deployment failed: {result.get('error')}")
        print(result)

    await agent.close()
setup_multiple_live_agents() async

Example 4: Host multiple agents with individual live UIs

Source code in toolboxv2/mods/registry/demo_custom_messaging.py
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
async def setup_multiple_live_agents():
    """Example 4: Host multiple agents with individual live UIs"""
    app = get_app("MultiAgentLiveExample")
    isaa = app.get_mod("isaa")

    # Initialize ISAA
    await isaa.init_isaa()

    # Create different specialized agents
    agents_config = [
        {
            "name": "math_tutor",
            "system": "You are a mathematics tutor. Explain concepts step-by-step with live progress updates.",
            "public_name": "Live Math Tutor",
            "port": 8770
        },
        {
            "name": "code_helper",
            "system": "You are a coding assistant. Help debug and explain code with detailed progress tracking.",
            "public_name": "Live Code Assistant",
            "port": 8771
        },
        {
            "name": "creative_writer",
            "system": "You are a creative writer. Generate stories and content with live creative process updates.",
            "public_name": "Live Creative Writer",
            "port": 8772
        }
    ]

    hosted_agents = []

    # Create and host each agent
    for config in agents_config:
        # Create agent builder
        builder = isaa.get_agent_builder(config["name"])
        builder.with_system_message(config["system"])
        # builder.with_models(complex_llm_model="openrouter/openai/gpt-4o")

        # Register agent
        await isaa.register_agent(builder)

        # Get agent instance
        agent = await isaa.get_agent(config["name"])

        # Host with live UI - Progress wird automatisch eingerichtet
        result = await isaa.publish_and_host_agent(
            agent=agent,
            public_name=config["public_name"],
            description=f"Specialized agent: {config['public_name']} with live progress updates",
        )

        hosted_agents.append({
            'name': config["name"],
            'agent': agent,
            'result': result
        })

        app.print(f"🚀 {config['public_name']} live at: {result['ui_url']}")

    # Test all agents with live progress
    app.print("\n🧪 Testing all agents with live progress:")

    test_queries = [
        ("math_tutor", "Explain how to solve quadratic equations step by step"),
        ("code_helper", "Debug this Python function and explain the process"),
        ("creative_writer", "Write a short story about AI and humans working together")
    ]

    for agent_name, query in test_queries:
        agent_info = next(a for a in hosted_agents if a['name'] == agent_name)
        app.print(f"Testing {agent_name} - watch live progress in UI...")

        try:
            result = await agent_info['agent'].a_run(query, session_id=f"test_{agent_name}")
            app.print(f"✅ {agent_name} completed - live progress was shown!")
        except Exception as e:
            app.print(f"❌ {agent_name} failed: {e}")

    # Keep all agents running
    try:
        while True:
            await asyncio.sleep(60)
            app.print("💓 All agents live and ready")
            for agent_info in hosted_agents:
                app.print(f"   • {agent_info['name']}: {agent_info['result']['ui_url']}")
    except KeyboardInterrupt:
        app.print("Shutting down all live agents...")
        for agent_info in hosted_agents:
            await agent_info['agent'].close()

demo_registry

run_end_user_test() async

Simuliert einen externen Aufruf an die öffentliche API des Registry Servers.

Source code in toolboxv2/mods/registry/demo_registry.py
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
async def run_end_user_test():
    """Simuliert einen externen Aufruf an die öffentliche API des Registry Servers."""
    print("--- [USER] Warte darauf, dass der Agent publiziert wird... ---")
    await published_event.wait()
    print("--- [USER] Agent ist jetzt öffentlich. Starte Testaufruf in 3 Sekunden... ---")
    await asyncio.sleep(3)

    public_url = published_info.get("public_url")
    api_key = published_info.get("public_api_key")

    if not public_url or not api_key:
        print("--- [USER] FEHLER: Keine öffentlichen Agenten-Infos gefunden!", file=sys.stderr)
        return

    print(f"--- [USER] Sende POST-Anfrage an: {public_url} ---")

    request_payload = {
        "query": "Hallo, weitergeleitete Welt!",
        "session_id": "ext-user-session-001"
    }

    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json"
    }

    async with aiohttp.ClientSession() as session:
        try:
            async with session.post(public_url, json=request_payload, headers=headers) as response:
                print(f"--- [USER] Antwort-Status: {response.status} ---")

                if response.status == 200:
                    print("--- [USER] Beginne mit dem Streamen der Antwort-Events: ---")
                    # Die Antwort ist application/json-seq, also lesen wir zeilenweise
                    async for line in response.content:
                        if line:
                            try:
                                data = json.loads(line)
                                event_type = data.get('event_type', 'unknown')
                                status = data.get('status', '...')
                                print(f"  [STREAM] Event: {event_type:<20} | Status: {status} {data}")

                                # Der finale Event enthält das Ergebnis
                                if event_type == "final_result":
                                    final_result = data.get('details', {}).get('result')
                                    print("\n--- [USER] Endgültiges Ergebnis erhalten: ---")
                                    print(f"  >>> {final_result}")

                            except json.JSONDecodeError:
                                print(f"  [STREAM] Konnte Zeile nicht als JSON parsen: {line.decode()}")
                else:
                    error_text = await response.text()
                    print(f"--- [USER] FEHLER vom Server: {error_text}", file=sys.stderr)
        except aiohttp.ClientConnectorError as e:
            print(f"--- [USER] VERBINDUNGSFEHLER: Konnte den Server nicht erreichen. Läuft er? Fehler: {e}",
                  file=sys.stderr)
run_local_client() async

Startet die zweite toolboxv2-Instanz als lokalen Client, der einen Agenten hostet.

Source code in toolboxv2/mods/registry/demo_registry.py
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
async def run_local_client():
    """Startet die zweite toolboxv2-Instanz als lokalen Client, der einen Agenten hostet."""
    print("--- [CLIENT] Initialisiere lokale Client Instanz ---")
    client_app = get_app("LocalClientInstance")

    # ISAA-Modul für diese Instanz holen und initialisieren
    isaa: ISAA_Tools = client_app.get_mod("isaa")
    await isaa.init_isaa()
    print("--- [CLIENT] ISAA initialisiert. ---")

    # --- Agenten erstellen ---
    print("--- [CLIENT] Erstelle einen einfachen 'EchoAgent'... ---")
    builder = isaa.get_agent_builder("EchoAgent")
    builder.with_system_message("You are an echo agent. Repeat the user's query exactly, but prefix it with 'Echo: '.")
    await isaa.register_agent(builder)

    # Agenten-Instanz holen (dieser Schritt ist nicht zwingend für das Publizieren per Name, aber gut zur Demo)
    echo_agent = await isaa.get_agent("EchoAgent")
    print(f"--- [CLIENT] 'EchoAgent' ({type(echo_agent).__name__}) erstellt. ---")

    # --- Agenten publizieren ---
    # Warten, bis der Server sicher läuft
    await asyncio.sleep(2)

    server_ws_url = "ws://127.0.0.1:8080/ws/registry/connect"
    print(f"--- [CLIENT] Publiziert 'EchoAgent' am Server: {server_ws_url} ---")

    # Die neue `publish_agent` Methode aufrufen
    reg_info = await isaa.host_agent_ui(
        agent=echo_agent,
        public_name="Public Echo Service",
        server_url=server_ws_url,
        description="A simple agent that echoes your input."
    )

    if reg_info:
        print("--- [CLIENT] Agent erfolgreich publiziert! Details erhalten: ---")
        print(f"  > Public URL: {reg_info.public_url}")
        print(f"  > API Key: {reg_info.public_api_key}")

        # Speichere die Info und signalisiere dem Endbenutzer-Task, dass er starten kann
        published_info.update(reg_info.model_dump())
        published_event.set()
    else:
        print("--- [CLIENT] FEHLER: Agenten-Publizierung fehlgeschlagen. ---", file=sys.stderr)

    # Hält diesen Task am Leben, um auf Weiterleitungsanfragen zu lauschen.
    await asyncio.Future()
run_registry_server() async

Startet die erste toolboxv2-Instanz als unseren öffentlichen Server.

Source code in toolboxv2/mods/registry/demo_registry.py
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
async def run_registry_server():
    """Startet die erste toolboxv2-Instanz als unseren öffentlichen Server."""
    print("--- [SERVER] Initialisiere Registry Server Instanz ---")

    # Holt sich eine App-Instanz. Das Laden des 'registry'-Moduls geschieht
    # automatisch durch die __init__.py-Struktur von toolboxv2.
    server_app = get_app("RegistryServerInstance")

    # Startet den actix-web Server auf Port 8080.
    # `blocking=False` ist entscheidend, damit asyncio weiterlaufen kann.
    server_app.start_server()

    print("--- [SERVER] Registry Server läuft auf http://127.0.0.1:8080 ---")
    print("--- [SERVER] Wartet auf eingehende Client-Verbindungen... ---")

    # Hält diesen Task am Leben, um den Server laufen zu lassen.
    await asyncio.Future()

server

broadcast_to_ui_clients(app, data) async

Broadcast updates to all connected UI clients.

Source code in toolboxv2/mods/registry/server.py
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
async def broadcast_to_ui_clients(app: App, data: dict[str, Any]):
    """Broadcast updates to all connected UI clients."""
    if not STATE.ui_clients:
        app.print("No active UI clients to broadcast to")
        return

    app.print(f"Broadcasting to {len(STATE.ui_clients)} UI clients: {data.get('event', 'unknown')}")

    dead_clients = set()
    successful_broadcasts = 0

    for ui_conn_id in STATE.ui_clients.copy():
        try:
            await app.ws_send(ui_conn_id, data)
            successful_broadcasts += 1
        except Exception as e:
            app.print(f"Failed to broadcast to UI client {ui_conn_id}: {e}")
            dead_clients.add(ui_conn_id)

    # Clean up dead connections
    for dead_client in dead_clients:
        STATE.ui_clients.discard(dead_client)

    app.print(f"Broadcast completed: {successful_broadcasts} successful, {len(dead_clients)} failed")
handle_agent_status_update(app, message) async

Handle agent status updates.

Source code in toolboxv2/mods/registry/server.py
191
192
193
194
195
196
197
198
199
200
201
async def handle_agent_status_update(app: App, message: WsMessage):
    """Handle agent status updates."""
    try:
        status_data = message.data
        await broadcast_to_ui_clients(app, {
            'event': 'agent_status_update',
            'data': status_data
        })

    except Exception as e:
        app.print(f"Agent status update error: {e}", error=True)
handle_execution_error(app, message) async

Handle execution errors.

Source code in toolboxv2/mods/registry/server.py
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
async def handle_execution_error(app: App, message: WsMessage):
    """Handle execution errors."""
    try:
        error = ExecutionError.model_validate(message.data)

        if error.request_id in STATE.pending_requests:
            await STATE.pending_requests[error.request_id].put(error)

        await broadcast_to_ui_clients(app, {
            'event': 'execution_error',
            'data': {
                'request_id': error.request_id,
                'error': error.error,
                'timestamp': asyncio.get_event_loop().time()
            }
        })

    except Exception as e:
        app.print(f"Execution error handling error: {e}", error=True)
handle_execution_result(app, message) async

Handle execution results.

Source code in toolboxv2/mods/registry/server.py
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
async def handle_execution_result(app: App, message: WsMessage):
    """Handle execution results."""
    try:
        result = ExecutionResult.model_validate(message.data)

        if result.request_id in STATE.pending_requests:
            await STATE.pending_requests[result.request_id].put(result)

        # Broadcast to UI clients
        await broadcast_to_ui_clients(app, {
            'event': 'execution_progress',
            'data': {
                'request_id': result.request_id,
                'payload': result.payload,
                'is_final': result.is_final,
                'timestamp': asyncio.get_event_loop().time()
            }
        })

    except Exception as e:
        app.print(f"Execution result error: {e}", error=True)
handle_registration(app, conn_id, session, message) async

Handle agent registration.

Source code in toolboxv2/mods/registry/server.py
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
async def handle_registration(app: App, conn_id: str, session: dict, message: WsMessage):
    """Handle agent registration."""
    try:
        reg_data = AgentRegistration.model_validate(message.data)
        agent_id = f"agent_{secrets.token_urlsafe(16)}"
        api_key = f"tbk_{secrets.token_urlsafe(32)}"

        STATE.client_agents.setdefault(conn_id, []).append(agent_id)
        STATE.agent_to_client[agent_id] = conn_id
        STATE.key_to_agent[api_key] = agent_id
        STATE.agent_details[agent_id] = reg_data.model_dump()

        base_url = os.getenv("APP_BASE_URL", "http://localhost:8080") or session.get('host', 'localhost:8080')
        if base_url == "localhost":
            base_url = "localhost:8080"
            app.print("APP_BASE_URL is localhost. Using default port 8080.")
        public_url = f"{base_url}/api/registry/run?public_agent_id={agent_id}"

        if not public_url.startswith('http'):
            public_url = f"http://{public_url}"

        response = AgentRegistered(
            public_name=reg_data.public_name,
            public_agent_id=agent_id,
            public_api_key=api_key,
            public_url=public_url,
        )

        # Send registration confirmation
        response_message = WsMessage(event='agent_registered', data=response.model_dump())
        await app.ws_send(conn_id, response_message.model_dump())

        # Notify UI clients
        await broadcast_to_ui_clients(app, {
            "event": "agent_registered",
            "data": {
                "public_agent_id": agent_id,
                "public_name": reg_data.public_name,
                "description": reg_data.description,
                "status": "online"
            }
        })

        app.print(f"Agent '{reg_data.public_name}' registered with ID: {agent_id}")

    except Exception as e:
        app.print(f"Registration error: {e}", error=True)
handle_ui_progress_update(app, message) async

Handle UI progress updates.

Source code in toolboxv2/mods/registry/server.py
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
async def handle_ui_progress_update(app: App, message: WsMessage):
    """Handle UI progress updates."""
    try:
        progress_data = message.data
        agent_id = progress_data.get('agent_id', 'unknown')

        # Store recent progress
        if agent_id not in STATE.recent_progress:
            STATE.recent_progress[agent_id] = []
        STATE.recent_progress[agent_id].append(progress_data)

        # Keep only last 50 events
        STATE.recent_progress[agent_id] = STATE.recent_progress[agent_id][-50:]

        # Broadcast to UI clients
        await broadcast_to_ui_clients(app, {
            "event": "live_progress_update",
            "data": progress_data
        })

    except Exception as e:
        app.print(f"UI progress update error: {e}", error=True)
on_disconnect(app, conn_id, session=None) async

Enhanced disconnect handler with comprehensive cleanup and UI notifications.

Source code in toolboxv2/mods/registry/server.py
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
async def on_disconnect(app: App, conn_id: str, session: dict = None):
    """Enhanced disconnect handler with comprehensive cleanup and UI notifications."""
    app.print(f"Registry client disconnected: {conn_id}")

    # Check if this is a UI client
    if conn_id in STATE.ui_clients:
        STATE.ui_clients.discard(conn_id)
        app.print(f"UI client {conn_id} removed from active clients")
        return

    # Handle agent client disconnection
    if conn_id in STATE.client_agents:
        agent_ids_to_cleanup = STATE.client_agents[conn_id].copy()

        for agent_id in agent_ids_to_cleanup:
            try:
                # Get agent details before removal for notification
                agent_details = STATE.agent_details.get(agent_id, {})
                agent_name = agent_details.get('public_name', 'Unknown')

                # Remove from all state dictionaries
                STATE.agent_to_client.pop(agent_id, None)
                STATE.agent_details.pop(agent_id, None)

                # Remove API key mapping
                key_to_remove = next((k for k, v in STATE.key_to_agent.items() if v == agent_id), None)
                if key_to_remove:
                    STATE.key_to_agent.pop(key_to_remove, None)

                # Clean up progress data
                STATE.recent_progress.pop(agent_id, None)

                # Clean up any pending requests for this agent by checking if queue exists and clearing it
                requests_to_cleanup = []
                for req_id in list(STATE.pending_requests.keys()):
                    try:
                        # Put error in queue to unblock any waiting requests
                        error_result = ExecutionError(
                            request_id=req_id,
                            error="Agent disconnected unexpectedly",
                            public_agent_id=agent_id
                        )
                        await STATE.pending_requests[req_id].put(error_result)
                        requests_to_cleanup.append(req_id)
                    except Exception as e:
                        app.print(f"Error cleaning up pending request {req_id}: {e}")

                # Remove cleaned up requests
                for req_id in requests_to_cleanup:
                    STATE.pending_requests.pop(req_id, None)

                # Notify UI clients about agent going offline (non-blocking)
                if agent_details:
                    asyncio.create_task(broadcast_to_ui_clients(app, {
                        "event": "agent_offline",
                        "data": {
                            "public_agent_id": agent_id,
                            "public_name": agent_name,
                            "status": "offline",
                            "timestamp": asyncio.get_event_loop().time()
                        }
                    }))

                app.print(f"Agent '{agent_name}' (ID: {agent_id}) unregistered and cleaned up")

            except Exception as e:
                app.print(f"Error during agent cleanup for {agent_id}: {e}", error=True)

        # Remove the client connection entry
        STATE.client_agents.pop(conn_id, None)

        app.print(f"Client {conn_id} fully disconnected and cleaned up ({len(agent_ids_to_cleanup)} agents removed)")
    else:
        app.print(f"Unknown client {conn_id} disconnected (no agents to clean up)")
on_message(app, conn_id, session, payload) async

Enhanced message handler with proper error handling.

Source code in toolboxv2/mods/registry/server.py
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
async def on_message(app: App, conn_id: str, session: dict, payload: dict):
    """Enhanced message handler with proper error handling."""
    try:
        # Ensure payload is a dict
        if isinstance(payload, str):
            payload = json.loads(payload)

        message = WsMessage.model_validate(payload)
        app.print(f"Registry received event: {message.event} from {conn_id}")

        if message.event == 'register':
            await handle_registration(app, conn_id, session, message)

        elif message.event == 'ui_progress_update':
            await handle_ui_progress_update(app, message)

        elif message.event == 'execution_result':
            await handle_execution_result(app, message)

        elif message.event == 'execution_error':
            await handle_execution_error(app, message)

        elif message.event == 'agent_status_update':
            await handle_agent_status_update(app, message)

        else:
            app.print(f"Unhandled event '{message.event}' from client {conn_id}")

    except Exception as e:
        app.print(f"Error processing WebSocket message: {e}", error=True)
register_ui_ws_handlers(app)

Register UI-specific WebSocket handlers.

Source code in toolboxv2/mods/registry/server.py
416
417
418
419
420
421
422
423
@export(mod_name=Name, websocket_handler="ui_connect")
def register_ui_ws_handlers(app: App):
    """Register UI-specific WebSocket handlers."""
    return {
        "on_connect": ui_on_connect,
        "on_message": ui_on_message,
        "on_disconnect": ui_on_disconnect,
    }
register_ws_handlers(app)

Register WebSocket handlers for the registry.

Source code in toolboxv2/mods/registry/server.py
406
407
408
409
410
411
412
413
@export(mod_name=Name, websocket_handler="connect")
def register_ws_handlers(app: App):
    """Register WebSocket handlers for the registry."""
    return {
        "on_connect": on_connect,
        "on_message": on_message,
        "on_disconnect": on_disconnect,
    }
run(app, public_agent_id, request) async

Public API endpoint to run agents.

Source code in toolboxv2/mods/registry/server.py
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
@export(mod_name=Name, api=True, version="1", request_as_kwarg=True, api_methods=['POST'])
async def run(app: App, public_agent_id: str, request: RequestData):
    """Public API endpoint to run agents."""
    if request is None:
        return Result.default_user_error(info="Failed to run agent: No request provided.")
    if not request.headers:
        return Result.default_user_error(info="Failed to run agent: No request headers provided.")

    auth_header = request.headers.authorization or request.headers.to_dict().get('authorization')

    if not auth_header or not auth_header.startswith('Bearer '):
        return Result.default_user_error("Authorization header missing or invalid.", exec_code=401)

    api_key = auth_header.split(' ')[1]

    if STATE.key_to_agent.get(api_key) != public_agent_id:
        return Result.default_user_error("Invalid API Key or Agent ID.", exec_code=403)

    conn_id = STATE.agent_to_client.get(public_agent_id)
    if not conn_id:
        return Result.default_internal_error("Agent is not currently connected/online.", exec_code=503)

    body = request.body
    request_id = f"req_{secrets.token_urlsafe(16)}"

    run_request = RunRequest(
        request_id=request_id,
        public_agent_id=public_agent_id,
        query=body.get('query', ''),
        session_id=body.get('session_id'),
        kwargs=body.get('kwargs', {})
    )

    response_queue = asyncio.Queue()
    STATE.pending_requests[request_id] = response_queue

    # Send run request to the client
    await app.ws_send(conn_id, WsMessage(event='run_request', data=run_request.model_dump()).model_dump())

    try:
        final_result = None
        while True:
            item = await asyncio.wait_for(response_queue.get(), timeout=120.0)

            if isinstance(item, ExecutionError):
                return Result.default_internal_error(
                    info=f"An error occurred during agent execution: {item.error}",
                    exec_code=500
                )

            if item.is_final:
                final_result = item.payload.get("details", {}).get("result")
                break

        return Result.json(data={"result": final_result})

    except TimeoutError:
        return Result.default_internal_error(
            info="The request timed out as the agent did not respond in time.",
            exec_code=504
        )
    finally:
        STATE.pending_requests.pop(request_id, None)
ui(app, public_agent_id=None) async

Serve the interactive 3-panel agent UI.

Source code in toolboxv2/mods/registry/server.py
491
492
493
494
495
496
@export(mod_name=Name, api=True, version="1", api_methods=['GET'])
async def ui(app: App, public_agent_id: str = None):
    """Serve the interactive 3-panel agent UI."""
    from ..isaa.ui import get_agent_ui_html
    html_content = get_agent_ui_html()
    return Result.html(data=html_content, row=True)
ui_on_connect(app, conn_id, session) async

UI Client connection.

Source code in toolboxv2/mods/registry/server.py
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
async def ui_on_connect(app: App, conn_id: str, session: dict):
    """UI Client connection."""
    app.print(f"UI Client connecting: {conn_id}")
    STATE.ui_clients.add(conn_id)
    app.print(f"UI Client connected: {conn_id} (Total: {len(STATE.ui_clients)})")

    # Send current agents list
    available_agents = []
    for agent_id, details in STATE.agent_details.items():
        if agent_id in STATE.agent_to_client:
            available_agents.append({
                "public_agent_id": agent_id,
                "public_name": details.get('public_name', 'Unknown'),
                "description": details.get('description', ''),
                "status": "online"
            })

    await app.ws_send(conn_id, {
        "event": "agents_list",
        "data": {"agents": available_agents}
    })
ui_on_disconnect(app, conn_id, session=None) async

UI Client Disconnection.

Source code in toolboxv2/mods/registry/server.py
400
401
402
403
async def ui_on_disconnect(app: App, conn_id: str, session: dict = None):
    """UI Client Disconnection."""
    app.print(f"UI Client disconnected: {conn_id}")
    STATE.ui_clients.discard(conn_id)
ui_on_message(app, conn_id, session, payload) async

UI Client Message Handler.

Source code in toolboxv2/mods/registry/server.py
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
async def ui_on_message(app: App, conn_id: str, session: dict, payload: dict):
    """UI Client Message Handler."""
    try:
        # Ensure payload is a dict
        if isinstance(payload, str):
            payload = json.loads(payload)

        event = payload.get('event')
        data = payload.get('data', {})

        if event == 'subscribe_agent':
            agent_id = data.get('public_agent_id')
            if agent_id in STATE.agent_details:
                if agent_id in STATE.recent_progress:
                    for progress_event in STATE.recent_progress[agent_id][-10:]:
                        await app.ws_send(conn_id, {
                            "event": "historical_progress",
                            "data": progress_event
                        })

                await app.ws_send(conn_id, {
                    "event": "subscription_confirmed",
                    "data": {"public_agent_id": agent_id}
                })

        elif event == 'chat_message':
            agent_id = data.get('public_agent_id')
            message_text = data.get('message')
            session_id = data.get('session_id', f'ui_{conn_id}')
            api_key = data.get('api_key')

            if not api_key or STATE.key_to_agent.get(api_key) != agent_id:
                await app.ws_send(conn_id, {
                    "event": "error",
                    "data": {"error": "Invalid or missing API Key"}
                })
                return

            if agent_id in STATE.agent_to_client:
                agent_conn_id = STATE.agent_to_client[agent_id]
                request_id = f"ui_req_{secrets.token_urlsafe(16)}"

                run_request = RunRequest(
                    request_id=request_id,
                    public_agent_id=agent_id,
                    query=message_text,
                    session_id=session_id,
                    kwargs={}
                )

                response_queue = asyncio.Queue()
                STATE.pending_requests[request_id] = response_queue

                await app.ws_send(agent_conn_id, WsMessage(
                    event='run_request',
                    data=run_request.model_dump()
                ).model_dump())

                await app.ws_send(conn_id, {
                    "event": "message_acknowledged",
                    "data": {"request_id": request_id, "agent_id": agent_id}
                })

    except Exception as e:
        app.print(f"UI message handling error: {e}", error=True)
        await app.ws_send(conn_id, {
            "event": "error",
            "data": {"error": str(e)}
        })

types

AgentRegistered

Bases: BaseModel

Server -> Client: Response after successful registration.

Source code in toolboxv2/mods/registry/types.py
14
15
16
17
18
19
class AgentRegistered(BaseModel):
    """Server -> Client: Response after successful registration."""
    public_name: str
    public_agent_id: str = Field(..., description="The unique public ID for the agent.")
    public_api_key: str = Field(..., description="The secret API key for public access.")
    public_url: str = Field(..., description="The full public URL to run the agent.")
AgentRegistration

Bases: BaseModel

Client -> Server: Payload to register a new agent.

Source code in toolboxv2/mods/registry/types.py
 9
10
11
12
class AgentRegistration(BaseModel):
    """Client -> Server: Payload to register a new agent."""
    public_name: str = Field(..., description="A user-friendly name for the agent.")
    description: str | None = Field(None, description="Optional description of the agent's capabilities.")
ExecutionError

Bases: BaseModel

Client -> Server: Reports an error during execution.

Source code in toolboxv2/mods/registry/types.py
35
36
37
38
class ExecutionError(BaseModel):
    """Client -> Server: Reports an error during execution."""
    request_id: str
    error: str
ExecutionResult

Bases: BaseModel

Client -> Server: A chunk of the execution result (for streaming).

Source code in toolboxv2/mods/registry/types.py
29
30
31
32
33
class ExecutionResult(BaseModel):
    """Client -> Server: A chunk of the execution result (for streaming)."""
    request_id: str
    payload: dict[str, Any] = Field(..., description="The ProgressEvent or final result as a dictionary.")
    is_final: bool = Field(False, description="True if this is the last message for this request.")
RunRequest

Bases: BaseModel

Server -> Client: Request to execute an agent.

Source code in toolboxv2/mods/registry/types.py
21
22
23
24
25
26
27
class RunRequest(BaseModel):
    """Server -> Client: Request to execute an agent."""
    request_id: str = Field(..., description="A unique ID for this specific execution request.")
    public_agent_id: str = Field(..., description="The ID of the agent to run.")
    query: str = Field(..., description="The main input/query for the agent.")
    session_id: str | None = Field(None, description="Session ID for maintaining context.")
    kwargs: dict[str, Any] = Field({}, description="Additional keyword arguments for the a_run method.")
WsMessage

Bases: BaseModel

A generic wrapper for all WebSocket messages.

Source code in toolboxv2/mods/registry/types.py
40
41
42
43
class WsMessage(BaseModel):
    """A generic wrapper for all WebSocket messages."""
    event: str
    data: dict[str, Any]

talk

TalkSession

Bases: BaseModel

Represents the state of a single voice conversation session.

Source code in toolboxv2/mods/talk.py
24
25
26
27
28
29
30
31
32
33
34
class TalkSession(BaseModel):
    """Represents the state of a single voice conversation session."""
    session_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
    user_id: str
    chat_session: ChatSession
    event_queue: asyncio.Queue = Field(default_factory=asyncio.Queue, exclude=True)
    # Task to track the running agent process, preventing concurrent requests
    agent_task: asyncio.Task | None = Field(default=None, exclude=True)

    class Config:
        arbitrary_types_allowed = True

Tools

Bases: MainTool

The main class for the Talk module, handling initialization, session management, and dependency loading.

Source code in toolboxv2/mods/talk.py
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
class Tools(MainTool):
    """
    The main class for the Talk module, handling initialization,
    session management, and dependency loading.
    """

    def __init__(self, app: App):
        # Initialize the MainTool with module-specific information
        self.version = VERSION
        self.name = MOD_NAME
        self.color = "CYAN"
        self.sessions: dict[str, TalkSession] = {}
        self.stt_func = None
        self.tts_func = None
        self.isaa_mod = None
        super().__init__(load=self.on_start, v=VERSION, name=MOD_NAME, tool={}, on_exit=self.on_exit)

    def on_start(self):
        """Initializes the Talk module, its dependencies (ISAA, AUDIO), and UI registration."""
        self.app.logger.info(f"Starting {self.name} v{self.version}...")

        # Get the ISAA module instance, which is a critical dependency
        self.isaa_mod = None#self.app.get_mod("isaa")
        if not self.isaa_mod:
            self.app.logger.error(
                f"{self.name}: ISAA module not found or failed to load. Voice assistant will not be functional.")
            return

        # Initialize STT and TTS services from the AUDIO module
        if hasattr(TBEF, "AUDIO") and self.app.get_mod("AUDIO"):
            self.stt_func = self.app.run_any(TBEF.AUDIO.STT_GENERATE, model="openai/whisper-small", row=True, device=0)
            self.tts_func = self.app.get_function(TBEF.AUDIO.SPEECH, state=False)[0]

            if self.stt_func and self.stt_func != "404":
                self.app.logger.info("Talk STT (whisper-small) is Online.")
            else:
                self.app.logger.warning("Talk STT function not available.")
                self.stt_func = None

            if self.tts_func and self.tts_func != "404":
                self.app.logger.info("Talk TTS function is Online.")
            else:
                self.app.logger.warning("Talk TTS function not available.")
                self.tts_func = None
        else:
            self.app.logger.warning("Talk module: AUDIO module features are not available or the module is not loaded.")

        if not all([self.stt_func, self.tts_func]):
            self.app.logger.error("Talk module cannot function without both STT and TTS services.")

        # Register the UI component with CloudM
        self.app.run_any(("CloudM", "add_ui"),
                         name=MOD_NAME, title="Voice Assistant", path=f"/api/{MOD_NAME}/ui",
                         description="Natural conversation with an AI assistant.", auth=True)
        self.app.logger.info(f"{self.name} UI registered with CloudM.")

    def on_exit(self):
        """Clean up resources, especially cancelling any active agent tasks."""
        for session in self.sessions.values():
            if session.agent_task and not session.agent_task.done():
                session.agent_task.cancel()
        self.app.logger.info(f"Closing {self.name} and cleaning up sessions.")
on_exit()

Clean up resources, especially cancelling any active agent tasks.

Source code in toolboxv2/mods/talk.py
94
95
96
97
98
99
def on_exit(self):
    """Clean up resources, especially cancelling any active agent tasks."""
    for session in self.sessions.values():
        if session.agent_task and not session.agent_task.done():
            session.agent_task.cancel()
    self.app.logger.info(f"Closing {self.name} and cleaning up sessions.")
on_start()

Initializes the Talk module, its dependencies (ISAA, AUDIO), and UI registration.

Source code in toolboxv2/mods/talk.py
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
def on_start(self):
    """Initializes the Talk module, its dependencies (ISAA, AUDIO), and UI registration."""
    self.app.logger.info(f"Starting {self.name} v{self.version}...")

    # Get the ISAA module instance, which is a critical dependency
    self.isaa_mod = None#self.app.get_mod("isaa")
    if not self.isaa_mod:
        self.app.logger.error(
            f"{self.name}: ISAA module not found or failed to load. Voice assistant will not be functional.")
        return

    # Initialize STT and TTS services from the AUDIO module
    if hasattr(TBEF, "AUDIO") and self.app.get_mod("AUDIO"):
        self.stt_func = self.app.run_any(TBEF.AUDIO.STT_GENERATE, model="openai/whisper-small", row=True, device=0)
        self.tts_func = self.app.get_function(TBEF.AUDIO.SPEECH, state=False)[0]

        if self.stt_func and self.stt_func != "404":
            self.app.logger.info("Talk STT (whisper-small) is Online.")
        else:
            self.app.logger.warning("Talk STT function not available.")
            self.stt_func = None

        if self.tts_func and self.tts_func != "404":
            self.app.logger.info("Talk TTS function is Online.")
        else:
            self.app.logger.warning("Talk TTS function not available.")
            self.tts_func = None
    else:
        self.app.logger.warning("Talk module: AUDIO module features are not available or the module is not loaded.")

    if not all([self.stt_func, self.tts_func]):
        self.app.logger.error("Talk module cannot function without both STT and TTS services.")

    # Register the UI component with CloudM
    self.app.run_any(("CloudM", "add_ui"),
                     name=MOD_NAME, title="Voice Assistant", path=f"/api/{MOD_NAME}/ui",
                     description="Natural conversation with an AI assistant.", auth=True)
    self.app.logger.info(f"{self.name} UI registered with CloudM.")

api_open_stream(self, request, session_id) async

Opens a Server-Sent Events (SSE) stream for a given session ID.

Source code in toolboxv2/mods/talk.py
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
@export(mod_name=MOD_NAME, api=True, name="stream", api_methods=['GET'], request_as_kwarg=True)
async def api_open_stream(self: Tools, request: RequestData, session_id: str) -> Result:
    """Opens a Server-Sent Events (SSE) stream for a given session ID."""
    if not session_id or session_id not in self.sessions:
        return Result.default_user_error(info="Invalid or expired session ID.", exec_code=404)

    session = self.sessions[session_id]
    queue = session.event_queue

    async def event_generator() -> AsyncGenerator[dict[str, Any], None]:
        self.app.logger.info(f"SSE stream opened for session {session_id}")
        await queue.put({"event": "connection_ready", "data": "Stream connected successfully."})
        try:
            while True:
                event_data = await queue.get()
                yield event_data
                queue.task_done()
        except asyncio.CancelledError:
            self.app.logger.info(f"SSE stream for session {session_id} cancelled by client.")
        finally:
            if session_id in self.sessions:
                if self.sessions[session_id].agent_task and not self.sessions[session_id].agent_task.done():
                    self.sessions[session_id].agent_task.cancel()
                del self.sessions[session_id]
                self.app.logger.info(f"Cleaned up and closed session {session_id}.")

    return Result.sse(stream_generator=event_generator())

api_process_audio(self, request, form_data) async

Receives audio, transcribes it, and starts the agent processing task.

Source code in toolboxv2/mods/talk.py
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
@export(mod_name=MOD_NAME, api=True, name="process_audio", api_methods=['POST'], request_as_kwarg=True)
async def api_process_audio(self: Tools, request: RequestData, form_data: dict) -> Result:
    """Receives audio, transcribes it, and starts the agent processing task."""
    if not self.stt_func:
        return Result.default_internal_error(info="Speech-to-text service is not available.")

    session_id = form_data.get('session_id')
    audio_file_data = form_data.get('audio_blob')

    if not session_id or session_id not in self.sessions:
        return Result.default_user_error(info="Invalid or missing session_id.", exec_code=400)

    session = self.sessions[session_id]

    if session.agent_task and not session.agent_task.done():
        return Result.default_user_error(info="Already processing a previous request.", exec_code=429)

    if not audio_file_data or 'content_base64' not in audio_file_data:
        return Result.default_user_error(info="Audio data is missing or in the wrong format.", exec_code=400)

    try:
        audio_bytes = base64.b64decode(audio_file_data['content_base64'])
        transcription_result = self.stt_func(audio_bytes)
        transcribed_text = transcription_result.get('text', '').strip()

        if not transcribed_text:
            await session.event_queue.put({"event": "error", "data": "Could not understand audio. Please try again."})
            return Result.ok(data={"message": "Transcription was empty."})

        await session.event_queue.put({"event": "transcription_update", "data": transcribed_text})

        voice_params = {
            "voice_index": int(form_data.get('voice_index', '0')),
            "provider": form_data.get('provider', 'piper'),
            "model_name": form_data.get('model_name', 'ryan')
        }

        # Start the background task; the request returns immediately.
        session.agent_task = asyncio.create_task(
            _run_agent_and_respond(self, session, transcribed_text, voice_params)
        )
        return Result.ok(data={"message": "Audio received and processing started."})

    except Exception as e:
        self.app.logger.error(f"Error processing audio for session {session_id}: {e}", exc_info=True)
        return Result.default_internal_error(info=f"Failed to process audio: {str(e)}")

api_start_session(self, request) async

Creates a new talk session for an authenticated user.

Source code in toolboxv2/mods/talk.py
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
@export(mod_name=MOD_NAME, api=True, name="start_session", api_methods=['POST'], request_as_kwarg=True)
async def api_start_session(self: Tools, request: RequestData) -> Result:
    """Creates a new talk session for an authenticated user."""
    user_id = await _get_user_uid(self.app, request)
    if not user_id:
        return Result.default_user_error(info="User authentication required.", exec_code=401)

    if not self.isaa_mod:
        return Result.default_internal_error(info="ISAA module is not available.")

    # Create a new ISAA ChatSession for conversation history
    chat_session = ChatSession(mem=self.isaa_mod.get_memory())
    session = TalkSession(user_id=user_id, chat_session=chat_session)
    self.sessions[session.session_id] = session

    self.app.logger.info(f"Started new talk session {session.session_id} for user {user_id}")
    return Result.json(data={"session_id": session.session_id})

get_main_ui(self, request)

Serves the main HTML and JavaScript UI for the Talk widget.

Source code in toolboxv2/mods/talk.py
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
@export(mod_name=MOD_NAME, name="ui", api=True, api_methods=['GET'], request_as_kwarg=True)
def get_main_ui(self: Tools, request: RequestData) -> Result:
    """Serves the main HTML and JavaScript UI for the Talk widget."""
    html_content = """
<!DOCTYPE html>
<html lang="en" data-theme="light">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>ToolBoxV2 - Voice Assistant</title>
    <link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Material+Symbols+Outlined:opsz,wght,FILL,GRAD@20..48,100..700,0..1,-50..200" />
    <style>
        body { font-family: sans-serif; background-color: var(--theme-bg); color: var(--theme-text); display: flex; justify-content: center; align-items: center; min-height: 100vh; margin: 0; }
        .container { display: flex; flex-direction: column; align-items: center; justify-content: center; width: 100%; max-width: 600px; padding: 20px; text-align: center; }
        .visualizer { width: 250px; height: 250px; background-color: var(--glass-bg); border-radius: 50%; position: relative; overflow: hidden; border: 3px solid var(--theme-border); box-shadow: inset 0 0 15px rgba(0,0,0,0.2); transition: border-color 0.3s, box-shadow 0.3s; }
        .visualizer.recording { border-color: #ef4444; }
        .visualizer.thinking { border-color: #3b82f6; animation: pulse 2s infinite; }
        .visualizer.speaking { border-color: #22c55e; }
        .particle { position: absolute; width: 8px; height: 8px; background-color: var(--theme-primary); border-radius: 50%; pointer-events: none; transition: all 0.1s; }
        #micButton { margin-top: 30px; width: 80px; height: 80px; border-radius: 50%; border: none; background-color: var(--theme-primary); color: white; cursor: pointer; display: flex; justify-content: center; align-items: center; box-shadow: 0 4px 10px rgba(0,0,0,0.2); transition: background-color 0.2s, transform 0.1s; }
        #micButton:active { transform: scale(0.95); }
        #micButton:disabled { background-color: #9ca3af; cursor: not-allowed; }
        #micButton .material-symbols-outlined { font-size: 40px; }
        #statusText { margin-top: 20px; min-height: 50px; font-size: 1.2em; color: var(--theme-text-muted); line-height: 1.5; }
        @keyframes pulse { 0% { box-shadow: inset 0 0 15px rgba(0,0,0,0.2), 0 0 0 0 rgba(59, 130, 246, 0.7); } 70% { box-shadow: inset 0 0 15px rgba(0,0,0,0.2), 0 0 0 15px rgba(59, 130, 246, 0); } 100% { box-shadow: inset 0 0 15px rgba(0,0,0,0.2), 0 0 0 0 rgba(59, 130, 246, 0); } }
    </style>
</head>
<body>
    <div class="container">
        <div class="visualizer" id="visualizer"></div>
        <p id="statusText">Press the microphone to start</p>
        <button id="micButton"><span class="material-symbols-outlined">hourglass_empty</span></button>
        <div class="options" style="margin-top: 20px;">
            <label for="voiceSelect">Voice:</label>
            <select id="voiceSelect">
                <option value='{"provider": "piper", "model_name": "ryan", "voice_index": 0}'>Ryan (EN)</option>
                <option value='{"provider": "piper", "model_name": "kathleen", "voice_index": 0}'>Kathleen (EN)</option>
                <option value='{"provider": "piper", "model_name": "karlsson", "voice_index": 0}'>Karlsson (DE)</option>
            </select>
        </div>
    </div>
    <script unSave="true">
    function initTalk() {
        const visualizer = document.getElementById('visualizer');
        const micButton = document.getElementById('micButton');
        const statusText = document.getElementById('statusText');
        const voiceSelect = document.getElementById('voiceSelect');

        const state = { sessionId: null, sseConnection: null, mediaRecorder: null, audioChunks: [], isRecording: false, isProcessing: false, currentAudio: null };
        let audioContext, analyser, particles = [];

        function setStatus(text, mode = 'idle') {
            statusText.textContent = text;
            visualizer.className = 'visualizer ' + mode;
        }

        function createParticles(num = 50) {
            visualizer.innerHTML = ''; particles = [];
            for (let i = 0; i < num; i++) {
                const p = document.createElement('div'); p.classList.add('particle');
                visualizer.appendChild(p);
                particles.push({ element: p, angle: Math.random() * Math.PI * 2, radius: 50 + Math.random() * 50, speed: 0.01 + Math.random() * 0.02 });
            }
        }

        function animateVisualizer() {
            if (analyser) {
                const dataArray = new Uint8Array(analyser.frequencyBinCount);
                analyser.getByteFrequencyData(dataArray);
                let average = dataArray.reduce((a, b) => a + b, 0) / dataArray.length;
                particles.forEach(p => {
                    p.angle += p.speed;
                    const scale = 1 + (average / 128);
                    p.element.style.transform = `translate(${Math.cos(p.angle) * p.radius * scale}px, ${Math.sin(p.angle) * p.radius * scale}px)`;
                });
            }
            requestAnimationFrame(animateVisualizer);
        }

        async function startSession() {
            if (state.sessionId) return;
            setStatus("Connecting...", 'thinking');
            micButton.disabled = true;
            try {
                const response = await TB.api.request('talk', 'start_session', {}, 'POST');
                if (response.error === 'none' && response.get()?.session_id) {
                    state.sessionId = response.get().session_id;
                    connectSse();
                } else {
                    setStatus(response.info?.help_text || "Failed to start session.", 'error');
                }
            } catch (e) {
                setStatus("Connection error.", 'error');
            }
        }

        function connectSse() {
            if (!state.sessionId) return;
            state.sseConnection = TB.sse.connect(`/sse/talk/stream?session_id=${state.sessionId}`, {
                onOpen: () => console.log("SSE Stream Open"),
                onError: () => setStatus("Connection lost.", 'error'),
                listeners: {
                    'connection_ready': (data) => { setStatus("Press the microphone to start"); micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; },
                    'transcription_update': (data) => { setStatus(`“${data}”`, 'thinking'); state.isProcessing = true; },
                    'agent_thought': (data) => setStatus(data, 'thinking'),
                    'agent_response_chunk': (data) => { if (statusText.textContent.startsWith('“')) statusText.textContent = ""; statusText.textContent += data; },
                    'audio_playback': (data) => playAudio(data.content, data.format),
                    'processing_complete': (data) => { state.isProcessing = false; setStatus(data); micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; },
                    'error': (data) => { state.isProcessing = false; setStatus(data, 'error'); micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; }
                }
            });
        }

        async function playAudio(base64, format) {
            setStatus("...", 'speaking');
            const blob = await (await fetch(`data:${format};base64,${base64}`)).blob();
            const url = URL.createObjectURL(blob);
            if (state.currentAudio) state.currentAudio.pause();
            state.currentAudio = new Audio(url);

            if (!audioContext) audioContext = new AudioContext();
            const source = audioContext.createMediaElementSource(state.currentAudio);
            if (!analyser) { analyser = audioContext.createAnalyser(); analyser.fftSize = 64; }
            source.connect(analyser);
            analyser.connect(audioContext.destination);

            state.currentAudio.play();
            state.currentAudio.onended = () => { setStatus("Finished speaking."); URL.revokeObjectURL(url); };
        }

        async function toggleRecording() {
            if (state.isProcessing) return;
            if (!state.sessionId) { await startSession(); return; }

            if (state.isRecording) {
                state.mediaRecorder.stop();
                micButton.disabled = true;
                micButton.innerHTML = '<span class="material-symbols-outlined">hourglass_top</span>';
                setStatus("Processing...", 'thinking');
            } else {
                if (!state.mediaRecorder) {
                    try {
                        const stream = await navigator.mediaDevices.getUserMedia({ audio: { sampleRate: 16000, channelCount: 1 } });
                        if (!audioContext) audioContext = new AudioContext();
                        const source = audioContext.createMediaStreamSource(stream);
                        if (!analyser) { analyser = audioContext.createAnalyser(); analyser.fftSize = 64; }
                        source.connect(analyser);

                        state.mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });
                        state.mediaRecorder.ondataavailable = e => state.audioChunks.push(e.data);
                        state.mediaRecorder.onstop = uploadAudio;
                    } catch (e) { setStatus("Could not access microphone.", 'error'); return; }
                }
                state.audioChunks = []; state.mediaRecorder.start(); state.isRecording = true;
                setStatus("Listening...", 'recording');
                micButton.innerHTML = '<span class="material-symbols-outlined">stop_circle</span>';
            }
        }

        async function uploadAudio() {
            state.isRecording = false; state.isProcessing = true;
            if (state.audioChunks.length === 0) { setStatus("No audio recorded."); state.isProcessing = false; micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>'; return; }
            const audioBlob = new Blob(state.audioChunks, { type: 'audio/webm;codecs=opus' });

            const formData = new FormData();
            formData.append('session_id', state.sessionId);
            formData.append('audio_blob', audioBlob, 'recording.webm');

            const voiceParams = JSON.parse(voiceSelect.value);
            for (const key in voiceParams) formData.append(key, voiceParams[key]);

            try {
                const response = await TB.api.request('talk', 'process_audio', formData, 'POST');
                if (response.error !== 'none') {
                    setStatus(response.info?.help_text || "Failed to process audio.", 'error');
                    state.isProcessing = false; micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>';
                }
            } catch(e) {
                 setStatus("Error sending audio.", 'error'); state.isProcessing = false; micButton.disabled = false; micButton.innerHTML = '<span class="material-symbols-outlined">mic</span>';
            }
        }

        micButton.addEventListener('click', toggleRecording);
        createParticles(); animateVisualizer();
        if (window.TB.isInitialized) startSession(); else window.TB.events.on('tbjs:initialized', startSession, { once: true });
    }
if (window.TB?.events) {
    if (window.TB.config?.get('appRootId')) { // A sign that TB.init might have run
         initTalk();
    } else {
        window.TB.events.on('tbjs:initialized', initTalk, { once: true });
    }
} else {
    // Fallback if TB is not even an object yet, very early load
    document.addEventListener('tbjs:initialized', initTalk, { once: true }); // Custom event dispatch from TB.init
}

    </script>
</body>
</html>"""
    return Result.html(data=html_content)

toolboxv2.flows_dict(s='.py', remote=False, dir_path=None, flows_dict_=None)

Source code in toolboxv2/flows/__init__.py
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
def flows_dict(s='.py', remote=False, dir_path=None, flows_dict_=None):

    if flows_dict_ is None:
        flows_dict_ = {}
    with Spinner("Loading flows"):
        # Erhalte den Pfad zum aktuellen Verzeichnis
        if dir_path is None:
            for ex_path in os.getenv("EXTERNAL_PATH_RUNNABLE", '').split(','):
                if not ex_path or len(ex_path) == 0:
                    continue
                flows_dict(s,remote,ex_path,flows_dict_)
            dir_path = os.path.dirname(os.path.realpath(__file__))
        to = time.perf_counter()
        # Iteriere über alle Dateien im Verzeichnis
        files = os.listdir(dir_path)
        l_files = len(files)
        for i, file_name in enumerate(files):
            if not file_name:
                continue
            with Spinner(f"{file_name} {i}/{l_files}"):
                if file_name == "__init__.py":
                    pass

                elif remote and s in file_name and file_name.endswith('.gist'):
                    name_f = os.path.splitext(file_name)[0]
                    name = name_f.split('.')[0]
                    url = name_f.split('.')[-1]
                    try:
                        module = GistLoader(f"{name}/{url}").load_module(name)
                    except Exception as e:
                        continue

                    if hasattr(module, 'run') and callable(module.run) and hasattr(module, 'NAME'):
                        flows_dict_[module.NAME] = module.run
                elif file_name.endswith('.py') and s in file_name:
                    name = os.path.splitext(file_name)[0]
                    spec = importlib.util.spec_from_file_location(name, os.path.join(dir_path, file_name))
                    module = importlib.util.module_from_spec(spec)
                    try:
                        spec.loader.exec_module(module)
                    except Exception:
                        continue

                    # Füge das Modul der Dictionary hinzu
                    if hasattr(module, 'run') and callable(module.run) and hasattr(module, 'NAME'):
                        flows_dict_[module.NAME] = module.run

        return flows_dict_

toolboxv2.TBEF

Automatic generated by ToolBox v = 0.1.22

Other Exposed Items

toolboxv2.ToolBox_over = 'root' module-attribute